TerraSync: improve copy-installed-files logic

This commit is contained in:
James Turner
2018-08-31 16:05:48 +01:00
parent 8c9fda9137
commit c192071f03
+47 -54
View File
@@ -182,17 +182,11 @@ class HTTPDirectory
ChildInfo(Type ty, const std::string & nameData, const std::string & hashData) : ChildInfo(Type ty, const std::string & nameData, const std::string & hashData) :
type(ty), type(ty),
name(nameData), name(nameData),
hash(hashData), hash(hashData)
sizeInBytes(0)
{ {
} }
ChildInfo(const ChildInfo& other) : ChildInfo(const ChildInfo& other) = default;
type(other.type),
name(other.name),
hash(other.hash),
sizeInBytes(other.sizeInBytes)
{ }
void setSize(const std::string & sizeData) void setSize(const std::string & sizeData)
{ {
@@ -206,7 +200,8 @@ class HTTPDirectory
Type type; Type type;
std::string name, hash; std::string name, hash;
size_t sizeInBytes; size_t sizeInBytes = 0;
SGPath path; // absolute path on disk
}; };
typedef std::vector<ChildInfo> ChildInfoList; typedef std::vector<ChildInfo> ChildInfoList;
@@ -273,44 +268,45 @@ public:
return; return;
} }
string_list indexNames = indexChildren(); char* buf = nullptr;
const_string_list_iterator nameIt = indexNames.begin(); size_t bufSize = 0;
for (; nameIt != indexNames.end(); ++nameIt) {
SGPath p(absolutePath());
p.append(*nameIt);
if (p.exists()) {
continue; // only copy if the file is missing entirely
}
ChildInfoList::iterator c = findIndexChild(*nameIt); for (const auto& child : children) {
if (c->type == ChildInfo::DirectoryType) { if (child.type != ChildInfo::FileType)
continue; // only care about files continue;
}
SGPath cp = _repository->installedCopyPath; if (child.path.exists())
cp.append(relativePath()); continue;
cp.append(*nameIt);
if (!cp.exists()) {
continue;
}
SGBinaryFile src(cp); SGPath cp = _repository->installedCopyPath;
SGBinaryFile dst(p); cp.append(relativePath());
src.open(SG_IO_IN); cp.append(child.name);
dst.open(SG_IO_OUT); if (!cp.exists()) {
continue;
}
char* buf = (char*) malloc(cp.sizeInBytes()); SGBinaryFile src(cp);
if (!buf) { SGBinaryFile dst(child.path);
continue; src.open(SG_IO_IN);
} dst.open(SG_IO_OUT);
src.read(buf, cp.sizeInBytes()); if (bufSize < cp.sizeInBytes()) {
dst.write(buf, cp.sizeInBytes()); bufSize = cp.sizeInBytes();
src.close(); free(buf);
dst.close(); buf = (char*) malloc(bufSize);
if (!buf) {
continue;
}
}
free(buf); src.read(buf, cp.sizeInBytes());
} dst.write(buf, cp.sizeInBytes());
src.close();
dst.close();
}
free(buf);
} }
void updateChildrenBasedOnHash() void updateChildrenBasedOnHash()
@@ -319,19 +315,20 @@ public:
copyInstalledChildren(); copyInstalledChildren();
string_list indexNames = indexChildren(), string_list toBeUpdated, orphans,
toBeUpdated, orphans; indexNames = indexChildren();
simgear::Dir d(absolutePath()); simgear::Dir d(absolutePath());
PathList fsChildren = d.children(0); PathList fsChildren = d.children(0);
for (const auto& child : fsChildren) { for (const auto& child : fsChildren) {
const auto& fileName = child.file(); const auto& fileName = child.file();
if ((fileName == ".dirindex") || (fileName == ".hashes")) { if ((fileName == ".dirindex") || (fileName == ".hashes")) {
continue; continue;
} }
ChildInfo info(child.isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType, fileName, ""); ChildInfo info(child.isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType,
fileName, "");
info.path = child;
std::string hash = hashForChild(info); std::string hash = hashForChild(info);
ChildInfoList::iterator c = findIndexChild(fileName); ChildInfoList::iterator c = findIndexChild(fileName);
@@ -428,19 +425,16 @@ public:
void didUpdateFile(const std::string& file, const std::string& hash, size_t sz) void didUpdateFile(const std::string& file, const std::string& hash, size_t sz)
{ {
// check hash matches what we expected // check hash matches what we expected
ChildInfoList::iterator it = findIndexChild(file); auto it = findIndexChild(file);
if (it == children.end()) { if (it == children.end()) {
SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file); SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file);
} else { } else {
SGPath fpath(absolutePath());
fpath.append(file);
if (it->hash != hash) { if (it->hash != hash) {
// we don't erase the file on a hash mismatch, becuase if we're syncing during the // we don't erase the file on a hash mismatch, becuase if we're syncing during the
// middle of a server-side update, the downloaded file may actually become valid. // middle of a server-side update, the downloaded file may actually become valid.
_repository->failedToUpdateChild(_relativePath, HTTPRepository::REPO_ERROR_CHECKSUM); _repository->failedToUpdateChild(_relativePath, HTTPRepository::REPO_ERROR_CHECKSUM);
} else { } else {
_repository->updatedFileContents(fpath, hash); _repository->updatedFileContents(it->path, hash);
_repository->totalDownloaded += sz; _repository->totalDownloaded += sz;
} // of hash matches } // of hash matches
} // of found in child list } // of found in child list
@@ -539,8 +533,8 @@ private:
continue; continue;
} }
children.push_back(ChildInfo(typeData == "f" ? ChildInfo::FileType : ChildInfo::DirectoryType, tokens[1], tokens[2])); children.emplace_back(ChildInfo(typeData == "f" ? ChildInfo::FileType : ChildInfo::DirectoryType, tokens[1], tokens[2]));
children.back().path = absolutePath() / tokens[1];
if (tokens.size() > 3) { if (tokens.size() > 3) {
children.back().setSize(tokens[3]); children.back().setSize(tokens[3]);
} }
@@ -572,8 +566,7 @@ private:
std::string hashForChild(const ChildInfo& child) const std::string hashForChild(const ChildInfo& child) const
{ {
SGPath p(absolutePath()); SGPath p(child.path);
p.append(child.name);
if (child.type == ChildInfo::DirectoryType) { if (child.type == ChildInfo::DirectoryType) {
p.append(".dirindex"); p.append(".dirindex");
} }