cmake/Source/cmArchiveWrite.cxx

/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
   file LICENSE.rst or https://cmake.org/licensing for details.  */
#include "cmArchiveWrite.h"

#include <cstdlib>
#include <ctime>
#include <iostream>
#include <limits>
#include <sstream>
#include <string>
#include <thread>

#include <cm/algorithm>
#include <cm/string_view>

#include <cm3p/archive.h>
#include <cm3p/archive_entry.h>

#include "cmsys/Directory.hxx"
#ifdef _WIN32
#  include "cmsys/Encoding.hxx"
#endif
#include "cmsys/FStream.hxx"

#include "cm_parse_date.h"

#include "cmStringAlgorithms.h"
#include "cmSystemTools.h"

#ifndef __LA_SSIZE_T
#  define __LA_SSIZE_T la_ssize_t
#endif

static std::string cm_archive_error_string(struct archive* a)
{
  char const* e = archive_error_string(a);
  return e ? e : "unknown error";
}

// Set path to be written to the archive.
static void cm_archive_entry_copy_pathname(struct archive_entry* e,
                                           char const* dest)
{
#ifdef _WIN32
  // libarchive converts our UTF-8 encoding to the archive's encoding.
  // `archive_entry_update_pathname_utf8` always populates the WCS form too.
  // It also populates the MBS form if possible, but we ignore conversion
  // failure because the archive formats support converting directly from
  // the WCS form to the archive's encoding without using the MBS form.
  archive_entry_update_pathname_utf8(e, dest);
#else
  // libarchive converts our locale's encoding to the archive's encoding.
  archive_entry_copy_pathname(e, dest);
#endif
}

// Set path used for filesystem access.
static void cm_archive_entry_copy_sourcepath(struct archive_entry* e,
                                             std::string const& file)
{
#ifdef _WIN32
  archive_entry_copy_sourcepath_w(e, cmsys::Encoding::ToWide(file).c_str());
#else
  archive_entry_copy_sourcepath(e, file.c_str());
#endif
}

class cmArchiveWrite::Entry
{
  struct archive_entry* Object;

public:
  Entry()
    : Object(archive_entry_new())
  {
  }
  ~Entry() { archive_entry_free(this->Object); }
  Entry(Entry const&) = delete;
  Entry& operator=(Entry const&) = delete;
  operator struct archive_entry *() { return this->Object; }
};

struct cmArchiveWrite::Callback
{
  // archive_write_callback
  static __LA_SSIZE_T Write(struct archive* /*unused*/, void* cd,
                            void const* b, size_t n)
  {
    cmArchiveWrite* self = static_cast<cmArchiveWrite*>(cd);
    if (self->Stream.write(static_cast<char const*>(b),
                           static_cast<std::streamsize>(n))) {
      return static_cast<__LA_SSIZE_T>(n);
    }
    return static_cast<__LA_SSIZE_T>(-1);
  }
};

cmArchiveWrite::cmArchiveWrite(std::ostream& os, Compress c,
                               std::string const& format,
                               std::string const& encoding,
                               int compressionLevel, int numThreads)
  : Stream(os)
  , Archive(archive_write_new())
  , Disk(archive_read_disk_new())
  , Format(format)
{
  // Upstream fixed an issue with their integer parsing in 3.4.0
  // which would cause spurious errors to be raised from `strtoull`.

  if (archive_write_set_format_by_name(this->Archive, format.c_str()) !=
      ARCHIVE_OK) {
    this->Error = cmStrCat("archive_write_set_format_by_name: ",
                           cm_archive_error_string(this->Archive));
    return;
  }

  bool is7zip = (format == "7zip");
  bool isZip = (format == "zip");
  bool isFormatSupportsCompressionNatively = (is7zip || isZip);

  if (numThreads < 1) {
    int upperLimit = (numThreads == 0) ? std::numeric_limits<int>::max()
                                       : std::abs(numThreads);

    numThreads =
      cm::clamp<int>(std::thread::hardware_concurrency(), 1, upperLimit);
  }

  std::string sNumThreads = std::to_string(numThreads);

  if (!isFormatSupportsCompressionNatively) {
    switch (c) {
      case CompressNone:
        if (archive_write_add_filter_none(this->Archive) != ARCHIVE_OK) {
          this->Error = cmStrCat("archive_write_add_filter_none: ",
                                 cm_archive_error_string(this->Archive));
          return;
        }
        break;
      case CompressCompress:
        if (archive_write_add_filter_compress(this->Archive) != ARCHIVE_OK) {
          this->Error = cmStrCat("archive_write_add_filter_compress: ",
                                 cm_archive_error_string(this->Archive));
          return;
        }
        break;
      case CompressGZip: {
        if (archive_write_add_filter_gzip(this->Archive) != ARCHIVE_OK) {
          this->Error = cmStrCat("archive_write_add_filter_gzip: ",
                                 cm_archive_error_string(this->Archive));
          return;
        }
        std::string source_date_epoch;
        cmSystemTools::GetEnv("SOURCE_DATE_EPOCH", source_date_epoch);
        if (!source_date_epoch.empty()) {
          // We're not able to specify an arbitrary timestamp for gzip.
          // The next best thing is to omit the timestamp entirely.
          if (archive_write_set_filter_option(
                this->Archive, "gzip", "timestamp", nullptr) != ARCHIVE_OK) {
            this->Error = cmStrCat("archive_write_set_filter_option: ",
                                   cm_archive_error_string(this->Archive));
            return;
          }
        }
      } break;
      case CompressBZip2:
        if (archive_write_add_filter_bzip2(this->Archive) != ARCHIVE_OK) {
          this->Error = cmStrCat("archive_write_add_filter_bzip2: ",
                                 cm_archive_error_string(this->Archive));
          return;
        }
        break;
      case CompressLZMA:
        if (archive_write_add_filter_lzma(this->Archive) != ARCHIVE_OK) {
          this->Error = cmStrCat("archive_write_add_filter_lzma: ",
                                 cm_archive_error_string(this->Archive));
          return;
        }
        break;
      case CompressXZ:
        if (archive_write_add_filter_xz(this->Archive) != ARCHIVE_OK) {
          this->Error = cmStrCat("archive_write_add_filter_xz: ",
                                 cm_archive_error_string(this->Archive));
          return;
        }

#if ARCHIVE_VERSION_NUMBER >= 3004000

#  ifdef _AIX
        // FIXME: Using more than 2 threads creates an empty archive.
        // Enforce this limit pending further investigation.
        if (numThreads > 2) {
          numThreads = 2;
          sNumThreads = std::to_string(numThreads);
        }
#  endif
        if (archive_write_set_filter_option(this->Archive, "xz", "threads",
                                            sNumThreads.c_str()) !=
            ARCHIVE_OK) {
          this->Error = cmStrCat("archive_compressor_xz_options: ",
                                 cm_archive_error_string(this->Archive));
          return;
        }
#endif

        break;
      case CompressZstd:
        if (archive_write_add_filter_zstd(this->Archive) != ARCHIVE_OK) {
          this->Error = cmStrCat("archive_write_add_filter_zstd: ",
                                 cm_archive_error_string(this->Archive));
          return;
        }

#if ARCHIVE_VERSION_NUMBER >= 3006000
        if (archive_write_set_filter_option(this->Archive, "zstd", "threads",
                                            sNumThreads.c_str()) !=
            ARCHIVE_OK) {
          this->Error = cmStrCat("archive_compressor_zstd_options: ",
                                 cm_archive_error_string(this->Archive));
          return;
        }
#endif
        break;
      case CompressPPMd:
        this->Error = cmStrCat("PPMd is not supported for ", format);
        return;
    }
  }

  // 7zip always uses UTF16-LE for the headers and doesn't support
  // header encoding specification.
  // arbsd can use the default encoding of the system only.
  if (!is7zip && format != "arbsd" && encoding != "OEM") {
    char const* formatForOptions = format == "paxr" ? "pax" : format.c_str();
    if (archive_write_set_format_option(this->Archive, formatForOptions,
                                        "hdrcharset",
                                        encoding.c_str()) != ARCHIVE_OK) {
      this->Error = cmStrCat("archive_write_set_format_option(hdrcharset): ",
                             cm_archive_error_string(this->Archive));
      return;
    }
  }

  if (isFormatSupportsCompressionNatively || compressionLevel != 0) {
    std::string compressionLevelStr = std::to_string(compressionLevel);
    std::string archiveFilterName;
    switch (c) {
      case CompressNone:
        if (is7zip || isZip) {
          archiveFilterName = "store";
        } else {
          // Nothing to do - the value should be empty
        }
        break;
      case CompressCompress:
        if (is7zip || isZip) {
          this->Error =
            cmStrCat("CompressCompress is not supported for ", format);
        } else {
          // Nothing to do - the value should be empty
        }
        break;
      case CompressGZip:
        if (is7zip || isZip) {
          archiveFilterName = "deflate";
        } else {
          archiveFilterName = "gzip";
        }
        break;
      case CompressBZip2:
#if ARCHIVE_VERSION_NUMBER < 3008000
        if (isZip) {
          this->Error = cmStrCat("BZip2 is not supported for ", format,
                                 ". Please, build CMake with libarchive 3.8.0 "
                                 "or newer if you want to use it.");
          return;
        }
#endif
        archiveFilterName = "bzip2";
        break;
      case CompressLZMA:
#if ARCHIVE_VERSION_NUMBER < 3008000
        if (isZip) {
          this->Error = cmStrCat("LZMA is not supported for ", format,
                                 ". Please, build CMake with libarchive 3.8.0 "
                                 "or newer if you want to use it.");
          return;
        }
#endif
        if (is7zip) {
          archiveFilterName = "lzma1";
        } else {
          archiveFilterName = "lzma";
        }
        break;
      case CompressXZ:
#if ARCHIVE_VERSION_NUMBER < 3008000
        if (isZip) {
          this->Error = cmStrCat("LZMA2 (XZ) is not supported for ", format,
                                 ". Please, build CMake with libarchive 3.8.0 "
                                 "or newer if you want to use it.");
          return;
        }
#endif
        if (is7zip) {
          archiveFilterName = "lzma2";
        } else {
          archiveFilterName = "xz";
        }
        break;
      case CompressZstd:
#if ARCHIVE_VERSION_NUMBER < 3008000
        if (is7zip || isZip) {
          this->Error = cmStrCat("Zstd is not supported for ", format,
                                 ". Please, build CMake with libarchive 3.8.0 "
                                 "or newer if you want to use it.");
          return;
        }
#endif
        archiveFilterName = "zstd";
        break;
      case CompressPPMd:
        if (is7zip) {
          archiveFilterName = "ppmd";
        } else {
          this->Error = cmStrCat("PPMd is not supported for ", format);
        }
        return;
    }

    if (isFormatSupportsCompressionNatively) {
      if (archiveFilterName.empty()) {
        this->Error = cmStrCat("Unknown compression method for ", format);
        return;
      }

      if (archive_write_set_format_option(
            this->Archive, format.c_str(), "compression",
            archiveFilterName.c_str()) != ARCHIVE_OK) {
        this->Error =
          cmStrCat("archive_write_set_format_option(compression): ",
                   cm_archive_error_string(this->Archive));
        return;
      }

#if ARCHIVE_VERSION_NUMBER >= 3008000
      if (archive_write_set_format_option(this->Archive, format.c_str(),
                                          "threads",
                                          sNumThreads.c_str()) != ARCHIVE_OK) {
        this->Error = cmStrCat("archive_write_set_format_option(threads): ",
                               cm_archive_error_string(this->Archive));
        return;
      }
#endif

      if (compressionLevel != 0) {
        if (archive_write_set_format_option(
              this->Archive, format.c_str(), "compression-level",
              compressionLevelStr.c_str()) != ARCHIVE_OK) {
          this->Error =
            cmStrCat("archive_write_set_format_option(compression-level): ",
                     cm_archive_error_string(this->Archive));
          return;
        }
      }
    } else if (compressionLevel != 0 && !archiveFilterName.empty()) {
      if (archive_write_set_filter_option(
            this->Archive, archiveFilterName.c_str(), "compression-level",
            compressionLevelStr.c_str()) != ARCHIVE_OK) {
        this->Error = cmStrCat("archive_write_set_filter_option: ",
                               cm_archive_error_string(this->Archive));
        return;
      }
    }
  }

#if !defined(_WIN32) || defined(__CYGWIN__)
  if (archive_read_disk_set_standard_lookup(this->Disk) != ARCHIVE_OK) {
    this->Error = cmStrCat("archive_read_disk_set_standard_lookup: ",
                           cm_archive_error_string(this->Archive));
    return;
  }
#endif

  // do not pad the last block!!
  if (archive_write_set_bytes_in_last_block(this->Archive, 1)) {
    this->Error = cmStrCat("archive_write_set_bytes_in_last_block: ",
                           cm_archive_error_string(this->Archive));
    return;
  }
}

bool cmArchiveWrite::Open()
{
  if (!this->Error.empty()) {
    return false;
  }
  if (archive_write_open(
        this->Archive, this, nullptr,
        reinterpret_cast<archive_write_callback*>(&Callback::Write),
        nullptr) != ARCHIVE_OK) {
    this->Error =
      cmStrCat("archive_write_open: ", cm_archive_error_string(this->Archive));
    return false;
  }
  return true;
}

bool cmArchiveWrite::SetExcludePatterns(
  std::vector<std::string> const& patterns)
{
  if (patterns.empty()) {
    return true;
  }
  if (!this->MatchObject) {
    this->MatchObject = archive_match_new();
    if (!this->MatchObject) {
      this->Error = "archive_match_new: out of memory";
      return false;
    }
  }
  // NOLINTNEXTLINE(readability-use-anyofallof)
  for (std::string const& pattern : patterns) {
    // Reject empty patterns ourselves.  libarchive would reject them too, but
    // only after allocating an error message on the match object that
    // archive_match_free() does not release.
    if (pattern.empty()) {
      this->Error = "exclusion pattern must not be empty";
      return false;
    }
    if (archive_match_exclude_pattern(this->MatchObject, pattern.c_str()) !=
        ARCHIVE_OK) {
      this->Error =
        cmStrCat("Failed to add to exclusion list: ", pattern, " (",
                 cm_archive_error_string(this->MatchObject), ')');
      return false;
    }
  }
  return true;
}

cmArchiveWrite::~cmArchiveWrite()
{
  if (this->MatchObject) {
    archive_match_free(this->MatchObject);
  }
  archive_read_free(this->Disk);
  archive_write_free(this->Archive);
}

bool cmArchiveWrite::Add(std::string path, size_t skip, char const* prefix,
                         bool recursive)
{
  if (!path.empty() && path.back() == '/') {
    path.erase(path.size() - 1);
  }
  this->AddPath(path, skip, prefix, recursive);
  return this->Okay();
}

bool cmArchiveWrite::AddPath(std::string const& path, size_t skip,
                             char const* prefix, bool recursive)
{
  // Skip paths whose archive entry name matches an exclusion pattern.  For a
  // directory this also prevents descending into it, pruning the subtree.
  if (this->MatchObject && skip < path.length()) {
    cm::string_view out = cm::string_view(path).substr(skip);
    std::string dest = cmStrCat(prefix ? prefix : "", out);
    Entry e;
    cm_archive_entry_copy_pathname(e, dest.c_str());
    int matched = archive_match_path_excluded(this->MatchObject, e);
    if (matched < 0) {
      this->Error = cmStrCat("archive_match_path_excluded: ",
                             cm_archive_error_string(this->MatchObject));
      return false;
    }
    if (matched > 0) {
      return true;
    }
  }
  if (path != "." || (this->Format != "zip" && this->Format != "7zip")) {
    if (!this->AddFile(path, skip, prefix)) {
      return false;
    }
  }
  if ((!cmSystemTools::FileIsDirectory(path) || !recursive) ||
      cmSystemTools::FileIsSymlink(path)) {
    return true;
  }
  cmsys::Directory d;
  if (d.Load(path)) {
    std::string next = cmStrCat(path, '/');
    if (next == "./" && (this->Format == "zip" || this->Format == "7zip")) {
      next.clear();
    }
    std::string::size_type end = next.size();
    unsigned long n = d.GetNumberOfFiles();
    for (unsigned long i = 0; i < n; ++i) {
      std::string const& file = d.GetFileName(i);
      if (file != "." && file != "..") {
        next.erase(end);
        next += file;
        if (!this->AddPath(next, skip, prefix)) {
          return false;
        }
      }
    }
  }
  return true;
}

bool cmArchiveWrite::AddFile(std::string const& file, size_t skip,
                             char const* prefix)
{
  this->Error = "";
  // Skip the file if we have no name for it.  This may happen on a
  // top-level directory, which does not need to be included anyway.
  if (skip >= file.length()) {
    return true;
  }
  cm::string_view out = cm::string_view(file).substr(skip);

  // Meta-data.
  std::string dest = cmStrCat(prefix ? prefix : "", out);
  if (this->Verbose) {
    std::cout << dest << "\n";
  }
  Entry e;
  cm_archive_entry_copy_sourcepath(e, file);
  cm_archive_entry_copy_pathname(e, dest.c_str());
  if (archive_read_disk_entry_from_file(this->Disk, e, -1, nullptr) !=
      ARCHIVE_OK) {
    this->Error =
      cmStrCat("Unable to read from file:\n  ", file, "\nbecause:\n  ",
               cm_archive_error_string(this->Disk));
    return false;
  }
  if (!this->MTime.empty()) {
    time_t now;
    time(&now);
    time_t t = cm_parse_date(now, this->MTime.c_str());
    if (t == -1) {
      this->Error = cmStrCat("unable to parse mtime '", this->MTime, '\'');
      return false;
    }
    archive_entry_set_mtime(e, t, 0);
  } else {
    std::string source_date_epoch;
    cmSystemTools::GetEnv("SOURCE_DATE_EPOCH", source_date_epoch);
    if (!source_date_epoch.empty()) {
      std::istringstream iss(source_date_epoch);
      time_t epochTime;
      iss >> epochTime;
      if (iss.eof() && !iss.fail()) {
        // Set all of the file times to the epoch time to handle archive
        // formats that include creation/access time.
        archive_entry_set_mtime(e, epochTime, 0);
        archive_entry_set_atime(e, epochTime, 0);
        archive_entry_set_ctime(e, epochTime, 0);
        archive_entry_set_birthtime(e, epochTime, 0);
      }
    }
  }

  // manages the uid/guid of the entry (if any)
  if (this->Uid.IsSet() && this->Gid.IsSet()) {
    archive_entry_set_uid(e, this->Uid.Get());
    archive_entry_set_gid(e, this->Gid.Get());
  }

  if (!this->Uname.empty() && !this->Gname.empty()) {
    archive_entry_set_uname(e, this->Uname.c_str());
    archive_entry_set_gname(e, this->Gname.c_str());
  }

  // manages the permissions
  if (this->Permissions.IsSet()) {
    archive_entry_set_perm(e, this->Permissions.Get());
  }

  if (this->PermissionsMask.IsSet()) {
    int perm = archive_entry_perm(e);
    archive_entry_set_perm(e, perm & this->PermissionsMask.Get());
  }

  // Clear acl and xattr fields not useful for distribution.
  archive_entry_acl_clear(e);
  archive_entry_xattr_clear(e);
  archive_entry_set_fflags(e, 0, 0);

  if (this->Format == "pax" || this->Format == "paxr") {
    // Sparse files are a GNU tar extension.
    // Do not use them in standard tar files.
    archive_entry_sparse_clear(e);
  }

  if (archive_write_header(this->Archive, e) != ARCHIVE_OK) {
    this->Error = cmStrCat("archive_write_header: ",
                           cm_archive_error_string(this->Archive));
    return false;
  }

  // do not copy content of symlink
  if (!archive_entry_symlink(e)) {
    // Content.
    if (size_t size = static_cast<size_t>(archive_entry_size(e))) {
      return this->AddData(file, size);
    }
  }
  return true;
}

bool cmArchiveWrite::AddData(std::string const& file, size_t size)
{
  cmsys::ifstream fin(file.c_str(), std::ios::in | std::ios::binary);
  if (!fin) {
    this->Error = cmStrCat("Error opening \"", file,
                           "\": ", cmSystemTools::GetLastSystemError());
    return false;
  }

  char buffer[16384];
  size_t nleft = size;
  while (nleft > 0) {
    using ssize_type = std::streamsize;
    size_t const nnext = nleft > sizeof(buffer) ? sizeof(buffer) : nleft;
    ssize_type const nnext_s = static_cast<ssize_type>(nnext);
    fin.read(buffer, nnext_s);
    // Some stream libraries (older HPUX) return failure at end of
    // file on the last read even if some data were read.  Check
    // gcount instead of trusting the stream error status.
    if (static_cast<size_t>(fin.gcount()) != nnext) {
      break;
    }
    if (archive_write_data(this->Archive, buffer, nnext) != nnext_s) {
      this->Error = cmStrCat("archive_write_data: ",
                             cm_archive_error_string(this->Archive));
      return false;
    }
    nleft -= nnext;
  }
  if (nleft > 0) {
    this->Error = cmStrCat("Error reading \"", file,
                           "\": ", cmSystemTools::GetLastSystemError());
    return false;
  }
  return true;
}