TerraSync: improve copy-installed-files logic
This commit is contained in:
@@ -182,17 +182,11 @@ class HTTPDirectory
|
|||||||
ChildInfo(Type ty, const std::string & nameData, const std::string & hashData) :
|
ChildInfo(Type ty, const std::string & nameData, const std::string & hashData) :
|
||||||
type(ty),
|
type(ty),
|
||||||
name(nameData),
|
name(nameData),
|
||||||
hash(hashData),
|
hash(hashData)
|
||||||
sizeInBytes(0)
|
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
ChildInfo(const ChildInfo& other) :
|
ChildInfo(const ChildInfo& other) = default;
|
||||||
type(other.type),
|
|
||||||
name(other.name),
|
|
||||||
hash(other.hash),
|
|
||||||
sizeInBytes(other.sizeInBytes)
|
|
||||||
{ }
|
|
||||||
|
|
||||||
void setSize(const std::string & sizeData)
|
void setSize(const std::string & sizeData)
|
||||||
{
|
{
|
||||||
@@ -206,7 +200,8 @@ class HTTPDirectory
|
|||||||
|
|
||||||
Type type;
|
Type type;
|
||||||
std::string name, hash;
|
std::string name, hash;
|
||||||
size_t sizeInBytes;
|
size_t sizeInBytes = 0;
|
||||||
|
SGPath path; // absolute path on disk
|
||||||
};
|
};
|
||||||
|
|
||||||
typedef std::vector<ChildInfo> ChildInfoList;
|
typedef std::vector<ChildInfo> ChildInfoList;
|
||||||
@@ -273,44 +268,45 @@ public:
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
string_list indexNames = indexChildren();
|
char* buf = nullptr;
|
||||||
const_string_list_iterator nameIt = indexNames.begin();
|
size_t bufSize = 0;
|
||||||
for (; nameIt != indexNames.end(); ++nameIt) {
|
|
||||||
SGPath p(absolutePath());
|
|
||||||
p.append(*nameIt);
|
|
||||||
if (p.exists()) {
|
|
||||||
continue; // only copy if the file is missing entirely
|
|
||||||
}
|
|
||||||
|
|
||||||
ChildInfoList::iterator c = findIndexChild(*nameIt);
|
for (const auto& child : children) {
|
||||||
if (c->type == ChildInfo::DirectoryType) {
|
if (child.type != ChildInfo::FileType)
|
||||||
continue; // only care about files
|
continue;
|
||||||
}
|
|
||||||
|
|
||||||
SGPath cp = _repository->installedCopyPath;
|
if (child.path.exists())
|
||||||
cp.append(relativePath());
|
continue;
|
||||||
cp.append(*nameIt);
|
|
||||||
if (!cp.exists()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
SGBinaryFile src(cp);
|
SGPath cp = _repository->installedCopyPath;
|
||||||
SGBinaryFile dst(p);
|
cp.append(relativePath());
|
||||||
src.open(SG_IO_IN);
|
cp.append(child.name);
|
||||||
dst.open(SG_IO_OUT);
|
if (!cp.exists()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
char* buf = (char*) malloc(cp.sizeInBytes());
|
SGBinaryFile src(cp);
|
||||||
if (!buf) {
|
SGBinaryFile dst(child.path);
|
||||||
continue;
|
src.open(SG_IO_IN);
|
||||||
}
|
dst.open(SG_IO_OUT);
|
||||||
|
|
||||||
src.read(buf, cp.sizeInBytes());
|
if (bufSize < cp.sizeInBytes()) {
|
||||||
dst.write(buf, cp.sizeInBytes());
|
bufSize = cp.sizeInBytes();
|
||||||
src.close();
|
free(buf);
|
||||||
dst.close();
|
buf = (char*) malloc(bufSize);
|
||||||
|
if (!buf) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
free(buf);
|
src.read(buf, cp.sizeInBytes());
|
||||||
}
|
dst.write(buf, cp.sizeInBytes());
|
||||||
|
src.close();
|
||||||
|
dst.close();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
free(buf);
|
||||||
}
|
}
|
||||||
|
|
||||||
void updateChildrenBasedOnHash()
|
void updateChildrenBasedOnHash()
|
||||||
@@ -319,19 +315,20 @@ public:
|
|||||||
|
|
||||||
copyInstalledChildren();
|
copyInstalledChildren();
|
||||||
|
|
||||||
string_list indexNames = indexChildren(),
|
string_list toBeUpdated, orphans,
|
||||||
toBeUpdated, orphans;
|
indexNames = indexChildren();
|
||||||
simgear::Dir d(absolutePath());
|
simgear::Dir d(absolutePath());
|
||||||
PathList fsChildren = d.children(0);
|
PathList fsChildren = d.children(0);
|
||||||
|
|
||||||
|
|
||||||
for (const auto& child : fsChildren) {
|
for (const auto& child : fsChildren) {
|
||||||
const auto& fileName = child.file();
|
const auto& fileName = child.file();
|
||||||
if ((fileName == ".dirindex") || (fileName == ".hashes")) {
|
if ((fileName == ".dirindex") || (fileName == ".hashes")) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
ChildInfo info(child.isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType, fileName, "");
|
ChildInfo info(child.isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType,
|
||||||
|
fileName, "");
|
||||||
|
info.path = child;
|
||||||
std::string hash = hashForChild(info);
|
std::string hash = hashForChild(info);
|
||||||
|
|
||||||
ChildInfoList::iterator c = findIndexChild(fileName);
|
ChildInfoList::iterator c = findIndexChild(fileName);
|
||||||
@@ -428,19 +425,16 @@ public:
|
|||||||
void didUpdateFile(const std::string& file, const std::string& hash, size_t sz)
|
void didUpdateFile(const std::string& file, const std::string& hash, size_t sz)
|
||||||
{
|
{
|
||||||
// check hash matches what we expected
|
// check hash matches what we expected
|
||||||
ChildInfoList::iterator it = findIndexChild(file);
|
auto it = findIndexChild(file);
|
||||||
if (it == children.end()) {
|
if (it == children.end()) {
|
||||||
SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file);
|
SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file);
|
||||||
} else {
|
} else {
|
||||||
SGPath fpath(absolutePath());
|
|
||||||
fpath.append(file);
|
|
||||||
|
|
||||||
if (it->hash != hash) {
|
if (it->hash != hash) {
|
||||||
// we don't erase the file on a hash mismatch, becuase if we're syncing during the
|
// we don't erase the file on a hash mismatch, becuase if we're syncing during the
|
||||||
// middle of a server-side update, the downloaded file may actually become valid.
|
// middle of a server-side update, the downloaded file may actually become valid.
|
||||||
_repository->failedToUpdateChild(_relativePath, HTTPRepository::REPO_ERROR_CHECKSUM);
|
_repository->failedToUpdateChild(_relativePath, HTTPRepository::REPO_ERROR_CHECKSUM);
|
||||||
} else {
|
} else {
|
||||||
_repository->updatedFileContents(fpath, hash);
|
_repository->updatedFileContents(it->path, hash);
|
||||||
_repository->totalDownloaded += sz;
|
_repository->totalDownloaded += sz;
|
||||||
} // of hash matches
|
} // of hash matches
|
||||||
} // of found in child list
|
} // of found in child list
|
||||||
@@ -539,8 +533,8 @@ private:
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
children.push_back(ChildInfo(typeData == "f" ? ChildInfo::FileType : ChildInfo::DirectoryType, tokens[1], tokens[2]));
|
children.emplace_back(ChildInfo(typeData == "f" ? ChildInfo::FileType : ChildInfo::DirectoryType, tokens[1], tokens[2]));
|
||||||
|
children.back().path = absolutePath() / tokens[1];
|
||||||
if (tokens.size() > 3) {
|
if (tokens.size() > 3) {
|
||||||
children.back().setSize(tokens[3]);
|
children.back().setSize(tokens[3]);
|
||||||
}
|
}
|
||||||
@@ -572,8 +566,7 @@ private:
|
|||||||
|
|
||||||
std::string hashForChild(const ChildInfo& child) const
|
std::string hashForChild(const ChildInfo& child) const
|
||||||
{
|
{
|
||||||
SGPath p(absolutePath());
|
SGPath p(child.path);
|
||||||
p.append(child.name);
|
|
||||||
if (child.type == ChildInfo::DirectoryType) {
|
if (child.type == ChildInfo::DirectoryType) {
|
||||||
p.append(".dirindex");
|
p.append(".dirindex");
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user