From 126f69434bca204e89721684867109d6a11e217e Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 4 Jul 2018 10:12:37 +0100 Subject: [PATCH] Refactor untar/unzip code This is to enable reuse in more places, especially scenery extraction and TerraSync --- simgear/io/test_untar.cxx | 80 +++++- simgear/io/untar.cxx | 532 ++++++++++++++++++++++++++---------- simgear/io/untar.hxx | 58 +++- simgear/io/zippy.zip | Bin 0 -> 811 bytes simgear/package/Install.cxx | 176 +----------- 5 files changed, 519 insertions(+), 327 deletions(-) create mode 100644 simgear/io/zippy.zip diff --git a/simgear/io/test_untar.cxx b/simgear/io/test_untar.cxx index 92c4ce29..b98b26e7 100644 --- a/simgear/io/test_untar.cxx +++ b/simgear/io/test_untar.cxx @@ -11,6 +11,7 @@ #include #include +#include #include @@ -31,7 +32,7 @@ void testTarGz() uint8_t* buf = (uint8_t*) alloca(8192); size_t bufSize = f.read((char*) buf, 8192); - SG_VERIFY(TarExtractor::isTarData(buf, bufSize)); + SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData); f.close(); } @@ -44,18 +45,89 @@ void testPlainTar() SGBinaryFile f(p); f.open(SG_IO_IN); - uint8_t* buf = (uint8_t*) alloca(8192); - size_t bufSize = f.read((char*) buf, 8192); + uint8_t* buf = (uint8_t*)alloca(8192); + size_t bufSize = f.read((char*) buf, 8192); - SG_VERIFY(TarExtractor::isTarData(buf, bufSize)); + SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData); f.close(); } +void testExtractStreamed() +{ + SGPath p = SGPath(SRC_DIR); + p.append("test.tar.gz"); + + SGBinaryFile f(p); + f.open(SG_IO_IN); + + SGPath extractDir = simgear::Dir::current().path() / "test_extract_streamed"; + simgear::Dir pd(extractDir); + pd.removeChildren(); + + ArchiveExtractor ex(extractDir); + + uint8_t* buf = (uint8_t*) alloca(128); + while (!f.eof()) { + size_t bufSize = f.read((char*) buf, 128); + ex.extractBytes(buf, bufSize); + } + + ex.flush(); + SG_VERIFY(ex.isAtEndOfArchive()); + SG_VERIFY(ex.hasError() == false); + + SG_VERIFY((extractDir / "testDir/hello.c").exists()); + SG_VERIFY((extractDir / "testDir/foo.txt").exists()); +} + +void testExtractLocalFile() +{ + +} + +void testExtractZip() +{ + SGPath p = SGPath(SRC_DIR); + p.append("zippy.zip"); + + SGBinaryFile f(p); + f.open(SG_IO_IN); + + SGPath extractDir = simgear::Dir::current().path() / "test_extract_zip"; + simgear::Dir pd(extractDir); + pd.removeChildren(); + + ArchiveExtractor ex(extractDir); + + uint8_t* buf = (uint8_t*)alloca(128); + while (!f.eof()) { + size_t bufSize = f.read((char*)buf, 128); + ex.extractBytes(buf, bufSize); + } + + ex.flush(); + SG_VERIFY(ex.isAtEndOfArchive()); + SG_VERIFY(ex.hasError() == false); + + SG_VERIFY((extractDir / "zippy/dirA/hello.c").exists()); + SG_VERIFY((extractDir / "zippy/bar.xml").exists()); + SG_VERIFY((extractDir / "zippy/long-named.json").exists()); +} + +void testPAXAttributes() +{ + +} + int main(int ac, char ** av) { testTarGz(); testPlainTar(); + testExtractStreamed(); + testExtractZip(); + + std::cout << "all tests passed" << std::endl; return 0; } diff --git a/simgear/io/untar.cxx b/simgear/io/untar.cxx index e6c8a2e8..be07863e 100644 --- a/simgear/io/untar.cxx +++ b/simgear/io/untar.cxx @@ -31,12 +31,78 @@ #include #include #include - +#include #include +#include +#include namespace simgear { + class ArchiveExtractorPrivate + { + public: + ArchiveExtractorPrivate(ArchiveExtractor* o) : + outer(o) + { + assert(outer); + } + + typedef enum { + INVALID = 0, + READING_HEADER, + READING_FILE, + READING_PADDING, + PRE_END_OF_ARCHVE, + END_OF_ARCHIVE, + ERROR_STATE, ///< states above this are error conditions + BAD_ARCHIVE, + BAD_DATA, + FILTER_STOPPED + } State; + + State state = INVALID; + ArchiveExtractor* outer = nullptr; + + virtual void extractBytes(const uint8_t* bytes, size_t count) = 0; + + virtual void flush() = 0; + + SGPath extractRootPath() + { + return outer->_rootPath; + } + + ArchiveExtractor::PathResult filterPath(std::string& pathToExtract) + { + return outer->filterPath(pathToExtract); + } + + + bool isSafePath(const std::string& p) const + { + if (p.empty()) { + return false; + } + + // reject absolute paths + if (p.at(0) == '/') { + return false; + } + + // reject paths containing '..' + size_t doubleDot = p.find(".."); + if (doubleDot != std::string::npos) { + return false; + } + + // on POSIX could use realpath to sanity check + return true; + } + }; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024; const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS; const int ZLIB_DECODE_GZIP_HEADER = 16; @@ -81,24 +147,14 @@ typedef struct #define FIFOTYPE '6' /* FIFO special */ #define CONTTYPE '7' /* reserved */ -class TarExtractorPrivate + const char PAX_GLOBAL_ATTRIBUTES = 'g'; + const char PAX_FILE_ATTRIBUTES = 'x'; + + +class TarExtractorPrivate : public ArchiveExtractorPrivate { public: - typedef enum { - INVALID = 0, - READING_HEADER, - READING_FILE, - READING_PADDING, - PRE_END_OF_ARCHVE, - END_OF_ARCHIVE, - ERROR_STATE, ///< states above this are error conditions - BAD_ARCHIVE, - BAD_DATA, - FILTER_STOPPED - } State; - - SGPath path; - State state; + union { UstarHeaderBlock header; uint8_t headerBytes[TAR_HEADER_BLOCK_SIZE]; @@ -109,17 +165,20 @@ public: size_t currentFileSize; z_stream zlibStream; uint8_t* zlibOutput; - bool haveInitedZLib; - bool uncompressedData; // set if reading a plain .tar (not tar.gz) + bool haveInitedZLib = false; + bool uncompressedData = false; // set if reading a plain .tar (not tar.gz) uint8_t* headerPtr; - TarExtractor* outer; bool skipCurrentEntry = false; - TarExtractorPrivate(TarExtractor* o) : - haveInitedZLib(false), - uncompressedData(false), - outer(o) + TarExtractorPrivate(ArchiveExtractor* o) : + ArchiveExtractorPrivate(o) { + memset(&zlibStream, 0, sizeof(z_stream)); + zlibOutput = (unsigned char*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE); + zlibStream.zalloc = Z_NULL; + zlibStream.zfree = Z_NULL; + zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE; + zlibStream.next_out = zlibOutput; } ~TarExtractorPrivate() @@ -168,6 +227,77 @@ public: state = newState; } + void extractBytes(const uint8_t* bytes, size_t count) override + { + zlibStream.next_in = (uint8_t*) bytes; + zlibStream.avail_in = count; + + if (!haveInitedZLib) { + // now we have data, see if we're dealing with GZ-compressed data or not + if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) { + // GZIP identification bytes + if (inflateInit2(&zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) { + SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed"); + state = TarExtractorPrivate::BAD_DATA; + return; + } + } else { + UstarHeaderBlock* header = (UstarHeaderBlock*)bytes; + if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) { + SG_LOG(SG_IO, SG_WARN, "didn't find tar magic in header"); + state = TarExtractorPrivate::BAD_DATA; + return; + } + + uncompressedData = true; + } + + haveInitedZLib = true; + setState(TarExtractorPrivate::READING_HEADER); + } // of init on first-bytes case + + if (uncompressedData) { + processBytes((const char*) bytes, count); + } else { + size_t writtenSize; + // loop, running zlib() inflate and sending output bytes to + // our request body handler. Keep calling inflate until no bytes are + // written, and ZLIB has consumed all available input + do { + zlibStream.next_out = zlibOutput; + zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE; + int result = inflate(&zlibStream, Z_NO_FLUSH); + if (result == Z_OK || result == Z_STREAM_END) { + // nothing to do + + } + else if (result == Z_BUF_ERROR) { + // transient error, fall through + } + else { + // _error = result; + SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << zlibStream.msg); + state = TarExtractorPrivate::BAD_DATA; + return; + } + + writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - zlibStream.avail_out; + if (writtenSize > 0) { + processBytes((const char*) zlibOutput, writtenSize); + } + + if (result == Z_STREAM_END) { + break; + } + } while ((zlibStream.avail_in > 0) || (writtenSize > 0)); + } // of Zlib-compressed data + } + + void flush() override + { + // no-op for tar files, we process everything greedily + } + void processHeader() { if (headerIsAllZeros()) { @@ -180,7 +310,7 @@ public: } if (strncmp(header.magic, TMAGIC, TMAGLEN) != 0) { - SG_LOG(SG_IO, SG_WARN, "magic is wrong"); + SG_LOG(SG_IO, SG_WARN, "Untar: magic is wrong"); state = BAD_ARCHIVE; return; } @@ -193,15 +323,15 @@ public: skipCurrentEntry = true; } - auto result = outer->filterPath(tarPath); - if (result == TarExtractor::Stop) { + auto result = filterPath(tarPath); + if (result == ArchiveExtractor::Stop) { setState(FILTER_STOPPED); return; - } else if (result == TarExtractor::Skipped) { + } else if (result == ArchiveExtractor::Skipped) { skipCurrentEntry = true; } - SGPath p = path / tarPath; + SGPath p = extractRootPath() / tarPath; if (header.typeflag == DIRTYPE) { if (!skipCurrentEntry) { Dir dir(p); @@ -260,133 +390,247 @@ public: return true; } - - bool isSafePath(const std::string& p) const - { - if (p.empty()) { - return false; - } - - // reject absolute paths - if (p.at(0) == '/') { - return false; - } - - // reject paths containing '..' - size_t doubleDot = p.find(".."); - if (doubleDot != std::string::npos) { - return false; - } - - // on POSIX could use realpath to sanity check - return true; - } }; -TarExtractor::TarExtractor(const SGPath& rootPath) : - d(new TarExtractorPrivate(this)) -{ +/////////////////////////////////////////////////////////////////////////////// - d->path = rootPath; - d->state = TarExtractorPrivate::INVALID; - - memset(&d->zlibStream, 0, sizeof(z_stream)); - d->zlibOutput = (unsigned char*) malloc(ZLIB_DECOMPRESS_BUFFER_SIZE); - d->zlibStream.zalloc = Z_NULL; - d->zlibStream.zfree = Z_NULL; - - d->zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE; - d->zlibStream.next_out = d->zlibOutput; +extern "C" { + void fill_memory_filefunc(zlib_filefunc_def*); } -TarExtractor::~TarExtractor() +class ZipExtractorPrivate : public ArchiveExtractorPrivate +{ +public: + std::string m_buffer; + + ZipExtractorPrivate(ArchiveExtractor* outer) : + ArchiveExtractorPrivate(outer) + { + + } + + ~ZipExtractorPrivate() + { + + } + + void extractBytes(const uint8_t* bytes, size_t count) override + { + // becuase the .zip central directory is at the end of the file, + // we have no choice but to simply buffer bytes here until flush() + // is called + m_buffer.append((const char*) bytes, count); + } + + void flush() override + { + zlib_filefunc_def memoryAccessFuncs; + fill_memory_filefunc(&memoryAccessFuncs); + + char bufferName[128]; + ::snprintf(bufferName, 128, "%p+%llx", m_buffer.data(), m_buffer.size()); + unzFile zip = unzOpen2(bufferName, &memoryAccessFuncs); + + const size_t BUFFER_SIZE = 32 * 1024; + void* buf = malloc(BUFFER_SIZE); + + try { + int result = unzGoToFirstFile(zip); + if (result != UNZ_OK) { + throw sg_exception("failed to go to first file in archive"); + } + + while (true) { + extractCurrentFile(zip, (char*)buf, BUFFER_SIZE); + if (state == FILTER_STOPPED) { + break; + } + + result = unzGoToNextFile(zip); + if (result == UNZ_END_OF_LIST_OF_FILE) { + break; + } + else if (result != UNZ_OK) { + throw sg_io_exception("failed to go to next file in the archive"); + } + } + state = END_OF_ARCHIVE; + } + catch (sg_exception&) { + state = BAD_ARCHIVE; + } + + free(buf); + unzClose(zip); + } + + void extractCurrentFile(unzFile zip, char* buffer, size_t bufferSize) + { + unz_file_info fileInfo; + unzGetCurrentFileInfo(zip, &fileInfo, + buffer, bufferSize, + NULL, 0, /* extra field */ + NULL, 0 /* comment field */); + + std::string name(buffer); + if (!isSafePath(name)) { + throw sg_format_exception("Bad zip path", name); + } + + auto filterResult = filterPath(name); + if (filterResult == ArchiveExtractor::Stop) { + state = FILTER_STOPPED; + return; + } + else if (filterResult == ArchiveExtractor::Skipped) { + return; + } + + if (fileInfo.uncompressed_size == 0) { + // assume it's a directory for now + // since we create parent directories when extracting + // a path, we're done here + return; + } + + int result = unzOpenCurrentFile(zip); + if (result != UNZ_OK) { + throw sg_io_exception("opening current zip file failed", sg_location(name)); + } + + sg_ofstream outFile; + bool eof = false; + SGPath path = extractRootPath() / name; + + // create enclosing directory heirarchy as required + Dir parentDir(path.dir()); + if (!parentDir.exists()) { + bool ok = parentDir.create(0755); + if (!ok) { + throw sg_io_exception("failed to create directory heirarchy for extraction", path); + } + } + + outFile.open(path, std::ios::binary | std::ios::trunc | std::ios::out); + if (outFile.fail()) { + throw sg_io_exception("failed to open output file for writing", path); + } + + while (!eof) { + int bytes = unzReadCurrentFile(zip, buffer, bufferSize); + if (bytes < 0) { + throw sg_io_exception("unzip failure reading curent archive", sg_location(name)); + } + else if (bytes == 0) { + eof = true; + } + else { + outFile.write(buffer, bytes); + } + } + + outFile.close(); + unzCloseCurrentFile(zip); + } +}; + +////////////////////////////////////////////////////////////////////////////// + +ArchiveExtractor::ArchiveExtractor(const SGPath& rootPath) : + _rootPath(rootPath) +{ +} + +ArchiveExtractor::~ArchiveExtractor() { } -void TarExtractor::extractBytes(const char* bytes, size_t count) +void ArchiveExtractor::extractBytes(const uint8_t* bytes, size_t count) { - if (d->state >= TarExtractorPrivate::ERROR_STATE) { + if (!d) { + _prebuffer.append((char*) bytes, count); + auto r = determineType((uint8_t*) _prebuffer.data(), _prebuffer.size()); + if (r == InsufficientData) { + return; + } + + if (r == TarData) { + d.reset(new TarExtractorPrivate(this)); + } + else if (r == ZipData) { + d.reset(new ZipExtractorPrivate(this)); + } + else { + SG_LOG(SG_IO, SG_ALERT, "Invcalid archive type"); + _invalidDataType = true; + return; + } + + // if hit here, we created the extractor. Feed the prefbuffer + // bytes through it + d->extractBytes((uint8_t*) _prebuffer.data(), _prebuffer.size()); + _prebuffer.clear(); + return; + } + + if (d->state >= ArchiveExtractorPrivate::ERROR_STATE) { return; } - d->zlibStream.next_in = (uint8_t*) bytes; - d->zlibStream.avail_in = count; - - if (!d->haveInitedZLib) { - // now we have data, see if we're dealing with GZ-compressed data or not - uint8_t* ubytes = (uint8_t*) bytes; - if ((ubytes[0] == 0x1f) && (ubytes[1] == 0x8b)) { - // GZIP identification bytes - if (inflateInit2(&d->zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) { - SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed"); - d->state = TarExtractorPrivate::BAD_DATA; - return; - } - } else { - UstarHeaderBlock* header = (UstarHeaderBlock*) bytes; - if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) { - SG_LOG(SG_IO, SG_WARN, "didn't find tar magic in header"); - d->state = TarExtractorPrivate::BAD_DATA; - return; - } - - d->uncompressedData = true; - } - - d->haveInitedZLib = true; - d->setState(TarExtractorPrivate::READING_HEADER); - } // of init on first-bytes case - - if (d->uncompressedData) { - d->processBytes(bytes, count); - } else { - size_t writtenSize; - // loop, running zlib() inflate and sending output bytes to - // our request body handler. Keep calling inflate until no bytes are - // written, and ZLIB has consumed all available input - do { - d->zlibStream.next_out = d->zlibOutput; - d->zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE; - int result = inflate(&d->zlibStream, Z_NO_FLUSH); - if (result == Z_OK || result == Z_STREAM_END) { - // nothing to do - - } else if (result == Z_BUF_ERROR) { - // transient error, fall through - } else { - // _error = result; - SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << d->zlibStream.msg); - d->state = TarExtractorPrivate::BAD_DATA; - return; - } - - writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - d->zlibStream.avail_out; - if (writtenSize > 0) { - d->processBytes((const char*) d->zlibOutput, writtenSize); - } - - if (result == Z_STREAM_END) { - break; - } - } while ((d->zlibStream.avail_in > 0) || (writtenSize > 0)); - } // of Zlib-compressed data + d->extractBytes(bytes, count); } -bool TarExtractor::isAtEndOfArchive() const +void ArchiveExtractor::flush() { - return (d->state == TarExtractorPrivate::END_OF_ARCHIVE); + if (!d) + return; + + d->flush(); } -bool TarExtractor::hasError() const +bool ArchiveExtractor::isAtEndOfArchive() const { - return (d->state >= TarExtractorPrivate::ERROR_STATE); + if (!d) + return false; + + return (d->state == ArchiveExtractorPrivate::END_OF_ARCHIVE); } -bool TarExtractor::isTarData(const uint8_t* bytes, size_t count) +bool ArchiveExtractor::hasError() const +{ + if (_invalidDataType) + return true; + + if (!d) + return false; + + return (d->state >= ArchiveExtractorPrivate::ERROR_STATE); +} + +ArchiveExtractor::DetermineResult ArchiveExtractor::determineType(const uint8_t* bytes, size_t count) +{ + // check for ZIP + if (count < 4) { + return InsufficientData; + } + + if (memcmp(bytes, "PK\x03\x04", 4) == 0) { + return ZipData; + } + + auto r = isTarData(bytes, count); + if ((r == TarData) || (r == InsufficientData)) + return r; + + return Invalid; +} + + +ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* bytes, size_t count) { if (count < 2) { - return false; + return InsufficientData; } UstarHeaderBlock* header = 0; @@ -404,21 +648,20 @@ bool TarExtractor::isTarData(const uint8_t* bytes, size_t count) if (inflateInit2(&z, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) { inflateEnd(&z); - return false; + return Invalid; } int result = inflate(&z, Z_SYNC_FLUSH); if (result != Z_OK) { SG_LOG(SG_IO, SG_WARN, "inflate failed:" << result); inflateEnd(&z); - return false; // not tar data + return Invalid; // not tar data } size_t written = 4096 - z.avail_out; if (written < TAR_HEADER_BLOCK_SIZE) { - SG_LOG(SG_IO, SG_WARN, "insufficient data for header"); inflateEnd(&z); - return false; + return InsufficientData; } header = reinterpret_cast(zlibOutput); @@ -426,22 +669,25 @@ bool TarExtractor::isTarData(const uint8_t* bytes, size_t count) } else { // uncompressed tar if (count < TAR_HEADER_BLOCK_SIZE) { - SG_LOG(SG_IO, SG_WARN, "insufficient data for header"); - return false; + return InsufficientData; } header = (UstarHeaderBlock*) bytes; } if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) { - SG_LOG(SG_IO, SG_WARN, "not a tar file"); - return false; + return Invalid; } - return true; + return TarData; } -auto TarExtractor::filterPath(std::string& pathToExtract) +void ArchiveExtractor::extractLocalFile(const SGPath& archiveFile) +{ + +} + +auto ArchiveExtractor::filterPath(std::string& pathToExtract) -> PathResult { SG_UNUSED(pathToExtract); diff --git a/simgear/io/untar.hxx b/simgear/io/untar.hxx index 0d963df2..1b3d9ccc 100644 --- a/simgear/io/untar.hxx +++ b/simgear/io/untar.hxx @@ -21,40 +21,68 @@ #include #include -#include // for uint8_t +#include + #include namespace simgear { -class TarExtractorPrivate; +class ArchiveExtractorPrivate; -class TarExtractor +class ArchiveExtractor { public: - TarExtractor(const SGPath& rootPath); - ~TarExtractor(); + ArchiveExtractor(const SGPath& rootPath); + ~ArchiveExtractor(); - static bool isTarData(const uint8_t* bytes, size_t count); + enum DetermineResult + { + Invalid, + InsufficientData, + TarData, + ZipData + }; - void extractBytes(const char* bytes, size_t count); + static DetermineResult determineType(const uint8_t* bytes, size_t count); + + /** + * @brief API to extract a local zip or tar.gz + */ + void extractLocalFile(const SGPath& archiveFile); + + /** + * @brief API to extract from memory - this can be called multiple + * times for streamking from a network socket etc + */ + void extractBytes(const uint8_t* bytes, size_t count); + + void flush(); bool isAtEndOfArchive() const; bool hasError() const; + enum PathResult { + Accepted, + Skipped, + Modified, + Stop + }; + protected: - enum PathResult { - Accepted, - Skipped, - Modified, - Stop - }; + virtual PathResult filterPath(std::string& pathToExtract); private: - friend class TarExtractorPrivate; - std::unique_ptr d; + static DetermineResult isTarData(const uint8_t* bytes, size_t count); + + friend class ArchiveExtractorPrivate; + std::unique_ptr d; + + SGPath _rootPath; + std::string _prebuffer; // store bytes before type is determined + bool _invalidDataType = false; }; } // of namespace simgear diff --git a/simgear/io/zippy.zip b/simgear/io/zippy.zip new file mode 100644 index 0000000000000000000000000000000000000000..5c0722d5d1b571e581c705d726d9e24aa7eea141 GIT binary patch literal 811 zcmWIWW@Zs#0D-659{Ye9P=XCeS7jCyRO$x+)o_7D9|6_Kw@WAR0(meDQInKdq*sxf zW0O&mn`39A52ir+AZEWrGn*T3c1mWEBa#L0feQO%{9ft-6@ai1s=|!aoSb~UWL~b! zyb^`n#LPSm5F@cDJy}N~IU})13&bvqj@9Jls^;ZV00LgFT7>g&08L+Cq9?BaG!=wJ zff(Zaocz3W-Mqxy)D*p};`}_QW~D?WD-$Cfh)GIGN>;{th9-t!BN>_O8E}Q03eZ>( z5P(onE4aW`GB7YQNH9G6DAgn(e1#E4Z%TR@4B`iPgUrPh)-ZDwfaZeOC?St*{FQi> zdXRDJi%umV(-7lvMG4G!kh5dp#)BgW+3@BgciKRP!f2Qukll_MjWEL%fCjgs8V-$A zWIN #include -extern "C" { - void fill_memory_filefunc (zlib_filefunc_def*); -} - namespace simgear { namespace pkg { @@ -57,9 +53,9 @@ public: throw sg_exception("no package download URLs"); } - if (m_owner->package()->properties()->hasChild("archive-type")) { - setArchiveTypeFromExtension(m_owner->package()->properties()->getStringValue("archive-type")); - } + // if (m_owner->package()->properties()->hasChild("archive-type")) { + // setArchiveTypeFromExtension(m_owner->package()->properties()->getStringValue("archive-type")); + //} // TODO randomise order of m_urls @@ -110,17 +106,17 @@ protected: Dir d(m_extractPath); d.create(0755); + m_extractor.reset(new ArchiveExtractor(m_extractPath)); memset(&m_md5, 0, sizeof(SG_MD5_CTX)); SG_MD5Init(&m_md5); } virtual void gotBodyData(const char* s, int n) { - m_buffer += std::string(s, n); - SG_MD5Update(&m_md5, (unsigned char*) s, n); - - m_downloaded = m_buffer.size(); - m_owner->installProgress(m_buffer.size(), responseLength()); + const uint8_t* ubytes = (uint8_t*) s; + SG_MD5Update(&m_md5, ubytes, n); + m_owner->installProgress(m_downloaded, responseLength()); + m_extractor->extractBytes(ubytes, n); } virtual void onDone() @@ -151,7 +147,8 @@ protected: return; } - if (!extract()) { + m_extractor->flush(); + if (m_extractor->hasError() || !m_extractor->isAtEndOfArchive()) { SG_LOG(SG_GENERAL, SG_WARN, "archive extraction failed"); doFailure(Delegate::FAIL_EXTRACT); return; @@ -207,151 +204,6 @@ protected: } private: - void setArchiveTypeFromExtension(const std::string& ext) - { - if (ext.empty()) - return; - - if (ext == "zip") { - m_archiveType = ZIP; - return; - } - - if ((ext == "tar.gz") || (ext == "tgz")) { - m_archiveType = TAR_GZ; - return; - } - } - - void extractCurrentFile(unzFile zip, char* buffer, size_t bufferSize) - { - unz_file_info fileInfo; - unzGetCurrentFileInfo(zip, &fileInfo, - buffer, bufferSize, - NULL, 0, /* extra field */ - NULL, 0 /* comment field */); - - std::string name(buffer); - // no absolute paths, no 'up' traversals - // we could also look for suspicious file extensions here (forbid .dll, .exe, .so) - if ((name[0] == '/') || (name.find("../") != std::string::npos) || (name.find("..\\") != std::string::npos)) { - throw sg_format_exception("Bad zip path", name); - } - - if (fileInfo.uncompressed_size == 0) { - // assume it's a directory for now - // since we create parent directories when extracting - // a path, we're done here - return; - } - - int result = unzOpenCurrentFile(zip); - if (result != UNZ_OK) { - throw sg_io_exception("opening current zip file failed", sg_location(name)); - } - - sg_ofstream outFile; - bool eof = false; - SGPath path(m_extractPath); - path.append(name); - - // create enclosing directory heirarchy as required - Dir parentDir(path.dir()); - if (!parentDir.exists()) { - bool ok = parentDir.create(0755); - if (!ok) { - throw sg_io_exception("failed to create directory heirarchy for extraction", path); - } - } - - outFile.open(path, std::ios::binary | std::ios::trunc | std::ios::out); - if (outFile.fail()) { - throw sg_io_exception("failed to open output file for writing", path); - } - - while (!eof) { - int bytes = unzReadCurrentFile(zip, buffer, bufferSize); - if (bytes < 0) { - throw sg_io_exception("unzip failure reading curent archive", sg_location(name)); - } else if (bytes == 0) { - eof = true; - } else { - outFile.write(buffer, bytes); - } - } - - outFile.close(); - unzCloseCurrentFile(zip); - } - - bool extract() - { - const std::string u(url()); - const size_t ul(u.length()); - - if (m_archiveType == AUTO_DETECT) { - if (u.rfind(".zip") == (ul - 4)) { - m_archiveType = ZIP; - } else if (u.rfind(".tar.gz") == (ul - 7)) { - m_archiveType = TAR_GZ; - } - // we will fall through to the error case now - } - - if (m_archiveType == ZIP) { - return extractUnzip(); - } else if (m_archiveType == TAR_GZ) { - return extractTar(); - } - - SG_LOG(SG_IO, SG_WARN, "unsupported archive format:" << u); - return false; - } - - bool extractUnzip() - { - bool result = true; - zlib_filefunc_def memoryAccessFuncs; - fill_memory_filefunc(&memoryAccessFuncs); - - char bufferName[128]; - snprintf(bufferName, 128, "%p+%lx", m_buffer.data(), m_buffer.size()); - unzFile zip = unzOpen2(bufferName, &memoryAccessFuncs); - - const size_t BUFFER_SIZE = 32 * 1024; - void* buf = malloc(BUFFER_SIZE); - - try { - int result = unzGoToFirstFile(zip); - if (result != UNZ_OK) { - throw sg_exception("failed to go to first file in archive"); - } - - while (true) { - extractCurrentFile(zip, (char*) buf, BUFFER_SIZE); - result = unzGoToNextFile(zip); - if (result == UNZ_END_OF_LIST_OF_FILE) { - break; - } else if (result != UNZ_OK) { - throw sg_io_exception("failed to go to next file in the archive"); - } - } - } catch (sg_exception& ) { - result = false; - } - - free(buf); - unzClose(zip); - return result; - } - - bool extractTar() - { - TarExtractor tx(m_extractPath); - tx.extractBytes(m_buffer.data(), m_buffer.size()); - return !tx.hasError() && tx.isAtEndOfArchive(); - } - void doFailure(Delegate::StatusCode aReason) { Dir dir(m_extractPath); @@ -364,19 +216,13 @@ private: m_owner->installResult(aReason); } - enum ArchiveType { - AUTO_DETECT = 0, - ZIP, - TAR_GZ - }; InstallRef m_owner; - ArchiveType m_archiveType = AUTO_DETECT; string_list m_urls; SG_MD5_CTX m_md5; - std::string m_buffer; SGPath m_extractPath; size_t m_downloaded; + std::unique_ptr m_extractor; }; ////////////////////////////////////////////////////////////////////