/*

* Copyright (C) by Klaas Freitag <freitag@owncloud.com>

*

* This library is free software; you can redistribute it and/or

* modify it under the terms of the GNU Lesser General Public

* License as published by the Free Software Foundation; either

* version 2.1 of the License, or (at your option) any later version.

*

* This library is distributed in the hope that it will be useful,

* but WITHOUT ANY WARRANTY; without even the implied warranty of

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

* Lesser General Public License for more details.

*

* You should have received a copy of the GNU Lesser General Public

* License along with this library; if not, write to the Free Software

* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

*/

#include <QCryptographicHash>

#include <QFile>

#include <QLoggingCategory>

#include <QStringList>

#include <QElapsedTimer>

#include <QUrl>

#include <QDir>

#include <sqlite3.h>

#include <cstring>

#include "common/asserts.h"

#include "common/checksums.h"

#include "common/preparedsqlquerymanager.h"

#include "common/syncjournaldb.h"

#include "common/version.h"

#include "filesystembase.h"

#include "version.h"

#include "common/c_jhash.h"

// SQL expression to check whether path.startswith(prefix + '/')

// Note: '/' + 1 == '0'

#define IS_PREFIX_PATH_OF(prefix, path) \

"(" path " > (" prefix "||'/') AND " path " < (" prefix "||'0'))"

#define IS_PREFIX_PATH_OR_EQUAL(prefix, path) \

"(" path " == " prefix " OR " IS_PREFIX_PATH_OF(prefix, path) ")"

namespace OCC {

Q_LOGGING_CATEGORY(lcDb, "sync.database", QtInfoMsg)

#define GET_FILE_RECORD_QUERY \

"SELECT path, inode, modtime, type, md5, fileid, remotePerm, filesize," \

" ignoredChildrenRemote, contentchecksumtype.name || ':' || contentChecksum" \

" FROM metadata" \

" LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id"

static void fillFileRecordFromGetQuery(SyncJournalFileRecord &rec, SqlQuery &query)

{

rec._path = query.baValue(0);

rec._inode = query.int64Value(1);

rec._modtime = query.int64Value(2);

rec._type = static_cast<ItemType>(query.intValue(3));

rec._etag = query.baValue(4);

rec._fileId = query.baValue(5);

rec._remotePerm = RemotePermissions::fromDbValue(query.baValue(6));

rec._fileSize = query.int64Value(7);

rec._serverHasIgnoredFiles = (query.intValue(8) > 0);

rec._checksumHeader = query.baValue(9);

}

static QByteArray defaultJournalMode(const QString &dbPath)

{

#if defined(Q_OS_WIN)

// See #2693: Some exFAT file systems seem unable to cope with the

// WAL journaling mode. They work fine with DELETE.

QString fileSystem = FileSystem::fileSystemForPath(dbPath);

qCInfo(lcDb) << "Detected filesystem" << fileSystem << "for" << dbPath;

if (fileSystem.contains(QLatin1String("FAT"))) {

qCInfo(lcDb) << "Filesystem contains FAT - using DELETE journal mode";

return "DELETE";

}

#elif defined(Q_OS_MAC)

if (dbPath.startsWith(QLatin1String("/Volumes/"))) {

qCInfo(lcDb) << "Mounted sync dir, do not use WAL for" << dbPath;

return "DELETE";

}

#else

Q_UNUSED(dbPath)

#endif

return "WAL";

}

SyncJournalDb::SyncJournalDb(const QString &dbFilePath, QObject *parent)

: QObject(parent)

, _dbFile(dbFilePath)

, _mutex(QMutex::Recursive)

, _transaction(0)

, _metadataTableIsEmpty(false)

{

// Allow forcing the journal mode for debugging

static QByteArray envJournalMode = qgetenv("OWNCLOUD_SQLITE_JOURNAL_MODE");

_journalMode = envJournalMode;

if (_journalMode.isEmpty()) {

_journalMode = defaultJournalMode(_dbFile);

}

}

QString SyncJournalDb::makeDbName(const QString &localPath,

const QString &infix)

{

const QString journalPath = QStringLiteral(".sync_") + infix + QStringLiteral(".db");

// If it exists already, the path is clearly usable

QFile file(QDir(localPath).filePath(journalPath));

if (file.exists()) {

return journalPath;

}

// Try to create a file there

if (file.open(QIODevice::ReadWrite)) {

// Ok, all good.

file.close();

file.remove();

return journalPath;

}

// Error during creation, just keep the original and throw errors later

qCWarning(lcDb) << "Could not find a writable database path" << file.fileName() << file.errorString();

return journalPath;

}

bool SyncJournalDb::maybeMigrateDb(const QString &localPath, const QString &absoluteJournalPath)

{

const QString oldDbName = localPath + QLatin1String(".csync_journal.db");

if (!FileSystem::fileExists(oldDbName)) {

return true;

}

const QString oldDbNameShm = oldDbName + QStringLiteral("-shm");

const QString oldDbNameWal = oldDbName + QStringLiteral("-wal");

const QString newDbName = absoluteJournalPath;

const QString newDbNameShm = newDbName + QStringLiteral("-shm");

const QString newDbNameWal = newDbName + QStringLiteral("-wal");

// Whenever there is an old db file, migrate it to the new db path.

// This is done to make switching from older versions to newer versions

// work correctly even if the user had previously used a new version

// and therefore already has an (outdated) new-style db file.

QString error;

if (FileSystem::fileExists(newDbName)) {

if (!FileSystem::remove(newDbName, &error)) {

qCWarning(lcDb) << "Database migration: Could not remove db file" << newDbName

<< "due to" << error;

return false;

}

}

if (FileSystem::fileExists(newDbNameWal)) {

if (!FileSystem::remove(newDbNameWal, &error)) {

qCWarning(lcDb) << "Database migration: Could not remove db WAL file" << newDbNameWal

<< "due to" << error;

return false;

}

}

if (FileSystem::fileExists(newDbNameShm)) {

if (!FileSystem::remove(newDbNameShm, &error)) {

qCWarning(lcDb) << "Database migration: Could not remove db SHM file" << newDbNameShm

<< "due to" << error;

return false;

}

}

if (!FileSystem::rename(oldDbName, newDbName, &error)) {

qCWarning(lcDb) << "Database migration: could not rename" << oldDbName

<< "to" << newDbName << ":" << error;

return false;

}

if (!FileSystem::rename(oldDbNameWal, newDbNameWal, &error)) {

qCWarning(lcDb) << "Database migration: could not rename" << oldDbNameWal

<< "to" << newDbNameWal << ":" << error;

return false;

}

if (!FileSystem::rename(oldDbNameShm, newDbNameShm, &error)) {

qCWarning(lcDb) << "Database migration: could not rename" << oldDbNameShm

<< "to" << newDbNameShm << ":" << error;

return false;

}

qCInfo(lcDb) << "Journal successfully migrated from" << oldDbName << "to" << newDbName;

return true;

}

bool SyncJournalDb::exists()

{

QMutexLocker locker(&_mutex);

return (!_dbFile.isEmpty() && QFile::exists(_dbFile));

}

QString SyncJournalDb::databaseFilePath() const

{

return _dbFile;

}

// Note that this does not change the size of the -wal file, but it is supposed to make

// the normal .db faster since the changes from the wal will be incorporated into it.

// Then the next sync (and the SocketAPI) will have a faster access.

void SyncJournalDb::walCheckpoint()

{

QElapsedTimer t;

t.start();

SqlQuery pragma1(_db);

pragma1.prepare("PRAGMA wal_checkpoint(FULL);");

if (pragma1.exec()) {

qCDebug(lcDb) << "took" << t.elapsed() << "msec";

}

}

void SyncJournalDb::startTransaction()

{

if (_transaction == 0) {

if (!_db.transaction()) {

qCWarning(lcDb) << "ERROR starting transaction:" << _db.error();

return;

}

_transaction = 1;

} else {

qCDebug(lcDb) << "Database Transaction is running, not starting another one!";

}

}

void SyncJournalDb::commitTransaction()

{

if (_transaction == 1) {

if (!_db.commit()) {

qCWarning(lcDb) << "ERROR committing to the database:" << _db.error();

return;

}

_transaction = 0;

} else {

qCDebug(lcDb) << "No database Transaction to commit";

}

}

bool SyncJournalDb::sqlFail(const QString &log, const SqlQuery &query)

{

commitTransaction();

qCWarning(lcDb) << "SQL Error" << log << query.error();

_db.close();

OC_ASSERT(false);

return false;

}

bool SyncJournalDb::checkConnect()

{

if (autotestFailCounter >= 0) {

if (!autotestFailCounter--) {

qCInfo(lcDb) << "Error Simulated";

return false;

}

}

if (_db.isOpen()) {

// Unfortunately the sqlite isOpen check can return true even when the underlying storage

// has become unavailable - and then some operations may cause crashes. See #6049

if (!QFile::exists(_dbFile)) {

qCWarning(lcDb) << "Database open, but file" << _dbFile << "does not exist";

close();

return false;

}

return true;

}

if (_dbFile.isEmpty()) {

qCWarning(lcDb) << "Database filename" << _dbFile << "is empty";

return false;

}

// The database file is created by this call (SQLITE_OPEN_CREATE)

if (!_db.openOrCreateReadWrite(_dbFile)) {

QString error = _db.error();

qCWarning(lcDb) << "Error opening the db:" << error;

return false;

}

if (!QFile::exists(_dbFile)) {

qCWarning(lcDb) << "Database file" << _dbFile << "does not exist";

return false;

}

SqlQuery pragma1(_db);

pragma1.prepare("SELECT sqlite_version();");

if (!pragma1.exec()) {

return sqlFail(QStringLiteral("SELECT sqlite_version()"), pragma1);

} else {

pragma1.next();

qCInfo(lcDb) << "sqlite3 version" << pragma1.stringValue(0);

}

// Set locking mode to avoid issues with WAL on Windows

static QByteArray locking_mode_env = qgetenv("OWNCLOUD_SQLITE_LOCKING_MODE");

if (locking_mode_env.isEmpty())

locking_mode_env = "EXCLUSIVE";

pragma1.prepare("PRAGMA locking_mode=" + locking_mode_env + ";");

if (!pragma1.exec()) {

return sqlFail(QStringLiteral("Set PRAGMA locking_mode"), pragma1);

} else {

pragma1.next();

qCInfo(lcDb) << "sqlite3 locking_mode=" << pragma1.stringValue(0);

}

pragma1.prepare("PRAGMA journal_mode=" + _journalMode + ";");

if (!pragma1.exec()) {

return sqlFail(QStringLiteral("Set PRAGMA journal_mode"), pragma1);

} else {

pragma1.next();

qCInfo(lcDb) << "sqlite3 journal_mode=" << pragma1.stringValue(0);

}

// For debugging purposes, allow temp_store to be set

static QByteArray env_temp_store = qgetenv("OWNCLOUD_SQLITE_TEMP_STORE");

if (!env_temp_store.isEmpty()) {

pragma1.prepare("PRAGMA temp_store = " + env_temp_store + ";");

if (!pragma1.exec()) {

return sqlFail(QStringLiteral("Set PRAGMA temp_store"), pragma1);

}

qCInfo(lcDb) << "sqlite3 with temp_store =" << env_temp_store;

}

// With WAL journal the NORMAL sync mode is safe from corruption,

// otherwise use the standard FULL mode.

QByteArray synchronousMode = "FULL";

if (QString::fromUtf8(_journalMode).compare(QStringLiteral("wal"), Qt::CaseInsensitive) == 0)

synchronousMode = "NORMAL";

pragma1.prepare("PRAGMA synchronous = " + synchronousMode + ";");

if (!pragma1.exec()) {

return sqlFail(QStringLiteral("Set PRAGMA synchronous"), pragma1);

} else {

qCInfo(lcDb) << "sqlite3 synchronous=" << synchronousMode;

}

pragma1.prepare("PRAGMA case_sensitive_like = ON;");

if (!pragma1.exec()) {

return sqlFail(QStringLiteral("Set PRAGMA case_sensitivity"), pragma1);

}

{

// Future version of the client (2.6) will have an index 'metadata_parent' which

// depends on a custom sqlite function which does not exist yet in 2.5.

// So make sure to remove the index if it exists, otherwise we will crash when inserting

// rows in the metadata database.

// The 2.6 client will anyway re-creates this index if it does not exist.

SqlQuery query(_db);

query.prepare("DROP INDEX IF EXISTS metadata_parent;");

if (!query.exec()) {

return sqlFail(QStringLiteral("updateMetadataTableStructure: remove index metadata_parent"), query);

}

}

sqlite3_create_function(_db.sqliteDb(), "parent_hash", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr,

[] (sqlite3_context *ctx,int, sqlite3_value **argv) {

auto text = reinterpret_cast<const char*>(sqlite3_value_text(argv[0]));

const char *end = std::strrchr(text, '/');

if (!end) end = text;

sqlite3_result_int64(ctx, c_jhash64(reinterpret_cast<const uint8_t*>(text),

end - text, 0));

}, nullptr, nullptr);

/* Because insert is so slow, we do everything in a transaction, and only need one call to commit */

startTransaction();

SqlQuery createQuery(_db);

createQuery.prepare("CREATE TABLE IF NOT EXISTS metadata("

"phash INTEGER(8),"

"pathlen INTEGER,"

"path VARCHAR(4096),"

"inode INTEGER,"

"uid INTEGER,"

"gid INTEGER,"

"mode INTEGER,"

"modtime INTEGER(8),"

"type INTEGER,"

"md5 VARCHAR(32)," /* This is the etag. Called md5 for compatibility */

// updateDatabaseStructure() will add

// fileid

// remotePerm

// filesize

// ignoredChildrenRemote

// contentChecksum

// contentChecksumTypeId

"PRIMARY KEY(phash)"

");");

#ifndef SQLITE_IOERR_SHMMAP

// Requires sqlite >= 3.7.7 but old CentOS6 has sqlite-3.6.20

// Definition taken from https://sqlite.org/c3ref/c_abort_rollback.html

#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8))

#endif

if (!createQuery.exec()) {

// In certain situations the io error can be avoided by switching

// to the DELETE journal mode, see #5723

if (_journalMode != "DELETE"

&& createQuery.errorId() == SQLITE_IOERR

&& sqlite3_extended_errcode(_db.sqliteDb()) == SQLITE_IOERR_SHMMAP) {

qCWarning(lcDb) << "IO error SHMMAP on table creation, attempting with DELETE journal mode";

_journalMode = "DELETE";

commitTransaction();

_db.close();

return checkConnect();

}

return sqlFail(QStringLiteral("Create table metadata"), createQuery);

}

createQuery.prepare("CREATE TABLE IF NOT EXISTS downloadinfo("

"path VARCHAR(4096),"

"tmpfile VARCHAR(4096),"

"etag VARCHAR(32),"

"errorcount INTEGER,"

"PRIMARY KEY(path)"

");");

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Create table downloadinfo"), createQuery);

}

createQuery.prepare("CREATE TABLE IF NOT EXISTS uploadinfo("

"path VARCHAR(4096),"

"chunk INTEGER,"

"transferid INTEGER,"

"errorcount INTEGER,"

"size INTEGER(8),"

"modtime INTEGER(8),"

"contentChecksum TEXT,"

"PRIMARY KEY(path)"

");");

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Create table uploadinfo"), createQuery);

}

// create the blacklist table.

createQuery.prepare("CREATE TABLE IF NOT EXISTS blacklist ("

"path VARCHAR(4096),"

"lastTryEtag VARCHAR[32],"

"lastTryModtime INTEGER[8],"

"retrycount INTEGER,"

"errorstring VARCHAR[4096],"

"PRIMARY KEY(path)"

");");

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Create table blacklist"), createQuery);

}

// create the selectivesync table.

createQuery.prepare("CREATE TABLE IF NOT EXISTS selectivesync ("

"path VARCHAR(4096),"

"type INTEGER"

");");

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Create table selectivesync"), createQuery);

}

// create the checksumtype table.

createQuery.prepare("CREATE TABLE IF NOT EXISTS checksumtype("

"id INTEGER PRIMARY KEY,"

"name TEXT UNIQUE"

");");

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Create table checksumtype"), createQuery);

}

// create the datafingerprint table.

createQuery.prepare("CREATE TABLE IF NOT EXISTS datafingerprint("

"fingerprint TEXT UNIQUE"

");");

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Create table datafingerprint"), createQuery);

}

// create the flags table.

createQuery.prepare("CREATE TABLE IF NOT EXISTS flags ("

"path TEXT PRIMARY KEY,"

"pinState INTEGER"

");");

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Create table flags"), createQuery);

}

// create the conflicts table.

createQuery.prepare("CREATE TABLE IF NOT EXISTS conflicts("

"path TEXT PRIMARY KEY,"

"baseFileId TEXT,"

"baseEtag TEXT,"

"baseModtime INTEGER"

");");

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Create table conflicts"), createQuery);

}

createQuery.prepare("CREATE TABLE IF NOT EXISTS version("

"major INTEGER(8),"

"minor INTEGER(8),"

"patch INTEGER(8),"

"custom VARCHAR(256)"

");");

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Create table version"), createQuery);

}

bool forceRemoteDiscovery = false;

SqlQuery versionQuery("SELECT major, minor, patch FROM version;", _db);

if (!versionQuery.next().hasData) {

forceRemoteDiscovery = true;

createQuery.prepare("INSERT INTO version VALUES (?1, ?2, ?3, ?4);");

const auto segments = OCC::Version::versionWithBuildNumber().segments();

for (int i = 0; i < segments.size(); ++i) {

createQuery.bindValue(i + 1, segments[i]);

}

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Update version"), createQuery);

}

} else {

int major = versionQuery.intValue(0);

int minor = versionQuery.intValue(1);

int patch = versionQuery.intValue(2);

if (major == 1 && minor == 8 && (patch == 0 || patch == 1)) {

qCInfo(lcDb) << "possibleUpgradeFromMirall_1_8_0_or_1 detected!";

forceRemoteDiscovery = true;

}

// - There was a bug in versions <2.3.0 that could lead to stale

// local files and a remote discovery will fix them.

// See #5190 #5242.

// - New remote HasZSyncMetadata permission added, invalidate cache

if (major == 2 && minor < 5) {

qCInfo(lcDb) << "upgrade from client < 2.5.0 detected! forcing remote discovery";

forceRemoteDiscovery = true;

}

// Not comparing the BUILD id here, correct?

if (QVersionNumber(major, minor, patch) != OCC::Version::version()) {

createQuery.prepare("UPDATE version SET major=?1, minor=?2, patch =?3, custom=?4 "

"WHERE major=?5 AND minor=?6 AND patch=?7;");

const auto segments = OCC::Version::versionWithBuildNumber().segments();

for (int i = 0; i < segments.size(); ++i) {

createQuery.bindValue(i + 1, segments[i]);

}

createQuery.bindValue(5, major);

createQuery.bindValue(6, minor);

createQuery.bindValue(7, patch);

if (!createQuery.exec()) {

return sqlFail(QStringLiteral("Update version"), createQuery);

}

}

}

commitInternal(QStringLiteral("checkConnect"));

bool rc = updateDatabaseStructure();

if (!rc) {

qCWarning(lcDb) << "Failed to update the database structure!";

}

/*

* If we are upgrading from a client version older than 1.5,

* we cannot read from the database because we need to fetch the files id and etags.

*

* If 1.8.0 caused missing data in the local tree, so we also don't read from DB

* to get back the files that were gone.

* In 1.8.1 we had a fix to re-get the data, but this one here is better

*/

if (forceRemoteDiscovery) {

forceRemoteDiscoveryNextSyncLocked();

}

const auto deleteDownloadInfo = _queryManager.get(PreparedSqlQueryManager::DeleteDownloadInfoQuery, QByteArrayLiteral("DELETE FROM downloadinfo WHERE path=?1"), _db);

if (!deleteDownloadInfo) {

return sqlFail(QStringLiteral("prepare _deleteDownloadInfoQuery"), *deleteDownloadInfo);

}

const auto deleteUploadInfoQuery = _queryManager.get(PreparedSqlQueryManager::DeleteUploadInfoQuery, QByteArrayLiteral("DELETE FROM uploadinfo WHERE path=?1"), _db);

if (!deleteUploadInfoQuery) {

return sqlFail(QStringLiteral("prepare _deleteUploadInfoQuery"), *deleteUploadInfoQuery);

}

QByteArray sql("SELECT lastTryEtag, lastTryModtime, retrycount, errorstring, lastTryTime, ignoreDuration, renameTarget, errorCategory, requestId "

"FROM blacklist WHERE path=?1");

if (Utility::fsCasePreserving()) {

// if the file system is case preserving we have to check the blacklist

// case insensitively

sql += " COLLATE NOCASE";

}

const auto getErrorBlacklistQuery = _queryManager.get(PreparedSqlQueryManager::GetErrorBlacklistQuery, sql, _db);

if (!getErrorBlacklistQuery) {

return sqlFail(QStringLiteral("prepare _getErrorBlacklistQuery"), *getErrorBlacklistQuery);

}

// don't start a new transaction now

commitInternal(QStringLiteral("checkConnect End"), false);

// This avoid reading from the DB if we already know it is empty

// thereby speeding up the initial discovery significantly.

_metadataTableIsEmpty = (getFileRecordCount() == 0);

// Hide 'em all!

FileSystem::setFileHidden(databaseFilePath(), true);

FileSystem::setFileHidden(databaseFilePath() + QStringLiteral("-wal"), true);

FileSystem::setFileHidden(databaseFilePath() + QStringLiteral("-shm"), true);

FileSystem::setFileHidden(databaseFilePath() + QStringLiteral("-journal"), true);

return rc;

}

void SyncJournalDb::close()

{

QMutexLocker locker(&_mutex);

qCInfo(lcDb) << "Closing DB" << _dbFile;

commitTransaction();

_db.close();

clearEtagStorageFilter();

_metadataTableIsEmpty = false;

}

bool SyncJournalDb::updateDatabaseStructure()

{

if (!updateMetadataTableStructure())

return false;

if (!updateErrorBlacklistTableStructure())

return false;

return true;

}

bool SyncJournalDb::updateMetadataTableStructure()

{

auto columns = tableColumns("metadata");

bool re = true;

// check if the file_id column is there and create it if not

if (columns.isEmpty()) {

return false;

}

if (columns.indexOf("fileid") == -1) {

SqlQuery query(_db);

query.prepare("ALTER TABLE metadata ADD COLUMN fileid VARCHAR(128);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: Add column fileid"), query);

re = false;

}

query.prepare("CREATE INDEX metadata_file_id ON metadata(fileid);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: create index fileid"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add fileid col"));

}

if (columns.indexOf("remotePerm") == -1) {

SqlQuery query(_db);

query.prepare("ALTER TABLE metadata ADD COLUMN remotePerm VARCHAR(128);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: add column remotePerm"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure (remotePerm)"));

}

if (columns.indexOf("filesize") == -1) {

SqlQuery query(_db);

query.prepare("ALTER TABLE metadata ADD COLUMN filesize BIGINT;");

if (!query.exec()) {

sqlFail(QStringLiteral("updateDatabaseStructure: add column filesize"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add filesize col"));

}

if (1) {

SqlQuery query(_db);

query.prepare("CREATE INDEX IF NOT EXISTS metadata_inode ON metadata(inode);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: create index inode"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add inode index"));

}

if (1) {

SqlQuery query(_db);

query.prepare("CREATE INDEX IF NOT EXISTS metadata_path ON metadata(path);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: create index path"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add path index"));

}

if (1) {

SqlQuery query(_db);

query.prepare("CREATE INDEX IF NOT EXISTS metadata_parent ON metadata(parent_hash(path));");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: create index parent"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add parent index"));

}

if (columns.indexOf("ignoredChildrenRemote") == -1) {

SqlQuery query(_db);

query.prepare("ALTER TABLE metadata ADD COLUMN ignoredChildrenRemote INT;");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: add ignoredChildrenRemote column"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add ignoredChildrenRemote col"));

}

if (columns.indexOf("contentChecksum") == -1) {

SqlQuery query(_db);

query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksum TEXT;");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: add contentChecksum column"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add contentChecksum col"));

}

if (columns.indexOf("contentChecksumTypeId") == -1) {

SqlQuery query(_db);

query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksumTypeId INTEGER;");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: add contentChecksumTypeId column"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add contentChecksumTypeId col"));

}

auto uploadInfoColumns = tableColumns("uploadinfo");

if (uploadInfoColumns.isEmpty())

return false;

if (!uploadInfoColumns.contains("contentChecksum")) {

SqlQuery query(_db);

query.prepare("ALTER TABLE uploadinfo ADD COLUMN contentChecksum TEXT;");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: add contentChecksum column"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add contentChecksum col for uploadinfo"));

}

auto conflictsColumns = tableColumns("conflicts");

if (conflictsColumns.isEmpty())

return false;

if (!conflictsColumns.contains("basePath")) {

SqlQuery query(_db);

query.prepare("ALTER TABLE conflicts ADD COLUMN basePath TEXT;");

if (!query.exec()) {

sqlFail(QStringLiteral("updateMetadataTableStructure: add basePath column"), query);

re = false;

}

}

return re;

}

bool SyncJournalDb::updateErrorBlacklistTableStructure()

{

auto columns = tableColumns("blacklist");

bool re = true;

if (columns.isEmpty()) {

return false;

}

if (columns.indexOf("lastTryTime") == -1) {

SqlQuery query(_db);

query.prepare("ALTER TABLE blacklist ADD COLUMN lastTryTime INTEGER(8);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateBlacklistTableStructure: Add lastTryTime fileid"), query);

re = false;

}

query.prepare("ALTER TABLE blacklist ADD COLUMN ignoreDuration INTEGER(8);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateBlacklistTableStructure: Add ignoreDuration fileid"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add lastTryTime, ignoreDuration cols"));

}

if (columns.indexOf("renameTarget") == -1) {

SqlQuery query(_db);

query.prepare("ALTER TABLE blacklist ADD COLUMN renameTarget VARCHAR(4096);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateBlacklistTableStructure: Add renameTarget"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add renameTarget col"));

}

if (columns.indexOf("errorCategory") == -1) {

SqlQuery query(_db);

query.prepare("ALTER TABLE blacklist ADD COLUMN errorCategory INTEGER(8);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateBlacklistTableStructure: Add errorCategory"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add errorCategory col"));

}

if (columns.indexOf("requestId") == -1) {

SqlQuery query(_db);

query.prepare("ALTER TABLE blacklist ADD COLUMN requestId VARCHAR(36);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateBlacklistTableStructure: Add requestId"), query);

re = false;

}

commitInternal(QStringLiteral("update database structure: add errorCategory col"));

}

SqlQuery query(_db);

query.prepare("CREATE INDEX IF NOT EXISTS blacklist_index ON blacklist(path collate nocase);");

if (!query.exec()) {

sqlFail(QStringLiteral("updateErrorBlacklistTableStructure: create index blacklit"), query);

re = false;

}

return re;

}

QVector<QByteArray> SyncJournalDb::tableColumns(const QByteArray &table)

{

QVector<QByteArray> columns;

if (!checkConnect()) {

return columns;

}

SqlQuery query("PRAGMA table_info('" + table + "');", _db);

if (!query.exec()) {

return columns;

}

while (query.next().hasData) {

columns.append(query.baValue(1));

}

qCDebug(lcDb) << "Columns in the current journal:" << columns;

return columns;

}

qint64 SyncJournalDb::getPHash(const QByteArray &file)

{

qint64 h;

int len = file.length();

h = c_jhash64((uint8_t *)file.data(), len, 0);

return h;

}

Result<void, QString> SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record)

{

SyncJournalFileRecord record = _record;

QMutexLocker locker(&_mutex);

if (!_etagStorageFilter.isEmpty()) {

// If we are a directory that should not be read from db next time, don't write the etag

QByteArray prefix = record._path + "/";

for (const auto &it : qAsConst(_etagStorageFilter)) {

if (it.startsWith(prefix)) {

qCInfo(lcDb) << "Filtered writing the etag of" << prefix << "because it is a prefix of" << it;

record._etag = "_invalid_";

break;

}

}

}

OC_ASSERT(!record._remotePerm.isNull());

qCInfo(lcDb) << "Updating file record for path:" << record._path << "inode:" << record._inode

<< "modtime:" << record._modtime << "type:" << record._type

<< "etag:" << record._etag << "fileId:" << record._fileId << "remotePerm:" << record._remotePerm.toString()

<< "fileSize:" << record._fileSize << "checksum:" << record._checksumHeader;

const qint64 phash = getPHash(record._path);

if (checkConnect()) {

int plen = record._path.length();

QByteArray etag(record._etag);

if (etag.isEmpty())

etag = "";

QByteArray fileId(record._fileId);

if (fileId.isEmpty())

fileId = "";

QByteArray remotePerm = record._remotePerm.toDbValue();

QByteArray checksumType, checksum;

parseChecksumHeader(record._checksumHeader, &checksumType, &checksum);

int contentChecksumTypeId = mapChecksumType(checksumType);

const auto query = _queryManager.get(PreparedSqlQueryManager::SetFileRecordQuery, QByteArrayLiteral("INSERT OR REPLACE INTO metadata "

"(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId) "

"VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16);"),

_db);

if (!query) {

return query->error();

}

query->bindValue(1, phash);

query->bindValue(2, plen);

query->bindValue(3, record._path);

query->bindValue(4, record._inode);

query->bindValue(5, 0); // uid Not used

query->bindValue(6, 0); // gid Not used

query->bindValue(7, 0); // mode Not used

query->bindValue(8, record._modtime);

query->bindValue(9, record._type);

query->bindValue(10, etag);

query->bindValue(11, fileId);

query->bindValue(12, remotePerm);

query->bindValue(13, record._fileSize);

query->bindValue(14, record._serverHasIgnoredFiles ? 1 : 0);

query->bindValue(15, checksum);

query->bindValue(16, contentChecksumTypeId);

if (!query->exec()) {

return query->error();

}

// Can't be true anymore.

_metadataTableIsEmpty = false;

return {};

} else {

qCWarning(lcDb) << "Failed to connect database.";

return tr("Failed to connect database."); // checkConnect failed.

}

}

// TODO: filename -> QBytearray?

bool SyncJournalDb::deleteFileRecord(const QString &filename, bool recursively)

{

QMutexLocker locker(&_mutex);

if (checkConnect()) {

// if (!recursively) {

// always delete the actual file.

{

const auto query = _queryManager.get(PreparedSqlQueryManager::DeleteFileRecordPhash, QByteArrayLiteral("DELETE FROM metadata WHERE phash=?1"), _db);

if (!query) {

return false;

}

const qint64 phash = getPHash(filename.toUtf8());

query->bindValue(1, phash);

if (!query->exec()) {

return false;

}

}

if (recursively) {

const auto query = _queryManager.get(PreparedSqlQueryManager::DeleteFileRecordRecursively, QByteArrayLiteral("DELETE FROM metadata WHERE " IS_PREFIX_PATH_OF("?1", "path")), _db);

if (!query)

return false;

query->bindValue(1, filename);

if (!query->exec()) {

return false;

}

}

return true;

} else {

qCWarning(lcDb) << "Failed to connect database.";

return false; // checkConnect failed.

}

}

bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRecord *rec)

{

QMutexLocker locker(&_mutex);

// Reset the output var in case the caller is reusing it.

Q_ASSERT(rec);

rec->_path.clear();

Q_ASSERT(!rec->isValid());

if (_metadataTableIsEmpty)

return true; // no error, yet nothing found (rec->isValid() == false)

if (!checkConnect())

return false;

if (!filename.isEmpty()) {

const auto query = _queryManager.get(PreparedSqlQueryManager::GetFileRecordQuery, QByteArrayLiteral(GET_FILE_RECORD_QUERY " WHERE phash=?1"), _db);

if (!query) {

return false;

}

query->bindValue(1, getPHash(filename));

if (!query->exec()) {

close();