Commit 3d64077f authored by Dario Freddi's avatar Dario Freddi

Make the whole delta implementatio work

Signed-off-by: default avatarDario Freddi <drf@kde.org>
parent 2f57dd41
......@@ -50,6 +50,7 @@ CREATE TABLE "groups" (
CREATE TABLE "deltas" (
"ID" INTEGER PRIMARY KEY AUTOINCREMENT,
"Package" INTEGER,
"Filename" TEXT,
"VersionFrom" TEXT,
"VersionTo" TEXT,
"MD5SUM" TEXT
......
......@@ -15,6 +15,7 @@
#include "akabeigroup.h"
#include "akabeipackage_p.h"
#include "akabeihelpers_p.h"
#include "akabeidelta_p.h"
#include <QtCore/QHash>
#include <QtCore/QtConcurrentRun>
......@@ -557,6 +558,121 @@ Package* Backend::loadPackageFromFile(const QString& path)
return retpackage;
}
Delta* Backend::loadDeltaFromFile(const QString& path)
{
int ret = ARCHIVE_OK;
struct archive *archive;
struct archive_entry *entry;
if (!QFile::exists(path)) {
return 0;
}
if((archive = archive_read_new()) == NULL) {
// RET_ERR(PM_ERR_LIBARCHIVE, NULL);
return 0;
}
archive_read_support_compression_all(archive);
archive_read_support_format_all(archive);
if (archive_read_open_filename(archive, path.toUtf8().data(), ARCHIVE_DEFAULT_BYTES_PER_BLOCK) != ARCHIVE_OK) {
// RET_ERR(PM_ERR_PKG_OPEN, NULL);
return 0;
}
Delta *retdelta = new Delta(QString(), QByteArray(), QByteArray());
retdelta->d_func()->pathToDelta = path;
/* If full is false, only read through the archive until we find our needed
* metadata. If it is true, read through the entire archive, which serves
* as a verfication of integrity and allows us to create the filelist. */
while((ret = archive_read_next_header(archive, &entry)) == ARCHIVE_OK) {
const char *entry_name = archive_entry_pathname(entry);
if(strcmp(entry_name, ".DELTAINFO") == 0) {
// Go and read from the archive
char fline[512];
while (Helpers::archive_file_readline(archive, fline) != 0) {
QString line(fline);
if (!line.contains('=') || line.startsWith('#')) {
// Skip this line, not interesting
continue;
}
line.remove('\n');
// Separate the two strings
QStringList splitted = line.split('=');
QString key = splitted.first();
// There might be an equal, so
splitted.removeAt(0);
QString value = splitted.join(QChar('='));
// Just remove all spaces ignorantly here
key = key.remove(' ');
// Recheck if it's commented over
if (key.startsWith('#')) {
// Pass by
}
// Strip whitespaces here instead
while (value.startsWith(' ')) {
value = value.remove(0, 1);
}
while (value.endsWith(' ')) {
value = value.remove(value.length() - 1, 1);
}
// Ok, now let's analyze
if (key == "pkgname") {
retdelta->d_func()->name = value;
} else if (key == "verfrom") {
retdelta->d_func()->vFrom = value.toUtf8();
} else if (key == "verto") {
retdelta->d_func()->vTo = value.toUtf8();
}
}
continue;
} else if(*entry_name == '.') {
/* for now, ignore all files starting with '.' that haven't
* already been handled (for future possibilities) */
} else {
// It can be only our delta
if (entry_name != path.split('/').last().remove(".tar.xz")) {
delete retdelta;
return 0;
}
}
if (archive_read_data_skip(archive)) {
delete retdelta;
return 0;
}
}
// Some checks
if (ret != ARCHIVE_EOF) {
delete retdelta;
return 0;
}
if (retdelta->targetName().isEmpty()) {
delete retdelta;
return 0;
}
// Now add md5sum and filename, of course
retdelta->d_func()->filename = path.split('/').last();
retdelta->d_func()->md5sum = Helpers::md5sumOfFile(path);
// We already computed md5 and validation, so set em up
retdelta->d_func()->_p_md5checked = true;
retdelta->d_func()->_p_validated = true;
return retdelta;
}
}
#include "akabeibackend.moc"
......@@ -17,6 +17,7 @@
class QUuid;
namespace Akabei {
class Delta;
class OperationRunner;
class Group;
class Package;
......@@ -58,6 +59,7 @@ class AKABEICORESHARED_EXPORT Backend : public QObject
OperationRunner *operationRunner();
Package *loadPackageFromFile(const QString &path);
Delta *loadDeltaFromFile(const QString &path);
public Q_SLOTS:
void initialize();
......
......@@ -14,7 +14,7 @@
namespace Akabei
{
Delta::Delta(const QString& targetName, const QString& versionFrom, const QString& versionTo)
Delta::Delta(const QString& targetName, const QByteArray& versionFrom, const QByteArray& versionTo)
: d_ptr(new DeltaPrivate(targetName, versionFrom, versionTo))
{
}
......@@ -48,12 +48,6 @@ QString Delta::filename() const
return d->filename;
}
QString Delta::filenameFrom() const
{
Q_D(const Delta);
return d->filenameFrom;
}
QByteArray Delta::md5sum() const
{
Q_D(const Delta);
......@@ -90,13 +84,13 @@ QString Delta::targetName() const
return d->name;
}
QString Delta::versionFrom() const
QByteArray Delta::versionFrom() const
{
Q_D(const Delta);
return d->vFrom;
}
QString Delta::versionTo() const
QByteArray Delta::versionTo() const
{
Q_D(const Delta);
return d->vTo;
......
......@@ -26,11 +26,10 @@ class Delta {
QByteArray md5sum() const;
QString targetName() const;
QString versionFrom() const;
QString versionTo() const;
QByteArray versionFrom() const;
QByteArray versionTo() const;
QString filename() const;
QString filenameFrom() const;
bool checkMD5Sum();
......@@ -43,10 +42,11 @@ class Delta {
void setPathToSource(const QString &path);
private:
Delta(const QString &targetName, const QString &versionFrom, const QString &versionTo);
Delta(const QString &targetName, const QByteArray &versionFrom, const QByteArray &versionTo);
DeltaPrivate * const d_ptr;
friend class Backend;
friend class Database;
friend class DatabasePrivate;
};
......
......@@ -18,27 +18,29 @@ namespace Akabei {
class DeltaPrivate {
public:
DeltaPrivate(const QString &n, const QString &v1, const QString &v2)
DeltaPrivate(const QString &n, const QByteArray &v1, const QByteArray &v2)
: name(n)
, vFrom(v1)
, vTo(v2)
, _p_md5checked(false)
, _p_validated(false)
{}
~DeltaPrivate() {}
QString name;
QByteArray md5sum;
QString vFrom;
QString vTo;
QByteArray vFrom;
QByteArray vTo;
QString filename;
QString filenameFrom;
QString pathToDelta;
QString pathToOldFile;
QString xdeltaApply;
bool _p_md5checked;
bool _p_validated;
};
}
......
......@@ -24,6 +24,7 @@
#include <archive_entry.h>
#include <qtemporaryfile.h>
#include <QProcess>
#include <akabeidelta.h>
const int MAX_DELTAS = 5;
const int MIN_PACKAGE_SIZE_FOR_DELTA = 3000000;
......@@ -33,6 +34,50 @@ bool packageVersionLessThan(Akabei::Package *p1, Akabei::Package *p2)
return p1->version() < p2->version();
}
bool performBind(sqlite3_stmt *stmt, const char *name, const QString &f)
{
// qDebug() << name << field;
QString field(f);
int result =
sqlite3_bind_text16(stmt, sqlite3_bind_parameter_index(stmt, name), field.utf16(),
(field.size()) * sizeof(QChar), SQLITE_TRANSIENT);
switch (result) {
case SQLITE_OK:
// Everything went fine and the handle is ready
return true;
default:
qDebug() << "Failed with " << result;
// Errors. Take action here
return false;
break;
}
// ??
return false;
}
bool performIntBind(sqlite3_stmt *stmt, const char *name, qint64 field)
{
// qDebug() << "Binding " << name << " to " << field << " index " << sqlite3_bind_parameter_index(stmt, name);
int result =
sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, name), field);
switch (result) {
case SQLITE_OK:
// Everything went fine and the handle is ready
return true;
default:
qDebug() << "Failed with " << result;
// Errors. Take action here
return false;
break;
}
// ??
return false;
}
CreateDb::CreateDb(const QString& db, const QStringList& targets, QObject* parent)
: QObject(parent)
, m_db(db)
......@@ -197,8 +242,8 @@ void CreateDb::start()
out << "Succeeded! Will use " << path << " as temporary file." << endl;
// Ok, now load all the targets
QMultiHash< QString, QPair< QByteArray, QByteArray > > deltas;
QHash< QString, QList< Akabei::Package* > > packages;
QMultiHash< QString, Akabei::Delta* > deltas;
int added = 0;
int addedDeltas = 0;
......@@ -215,19 +260,13 @@ void CreateDb::start()
++idx;
// Is it a delta?
if (target.endsWith(".delta.tar.xz")) {
// It is. Let's try having a look at what it is. The delta naming format is <name>-<vfrom>_to_<vto>.delta
QStringList targetSplitted = target.split('-');
QString tName = targetSplitted.first().split('/').last();
targetSplitted.removeAt(0);
QString versions = targetSplitted.join(QChar('-'));
versions.remove(".delta");
QByteArray vfrom = versions.split("_to_").first().toUtf8();
QByteArray vto = versions.split("_to_").last().toUtf8();
if (!tName.isEmpty() && !vfrom.isEmpty() && !vto.isEmpty()) {
deltas.insert(tName, qMakePair(vfrom, vto));
// It is. Let's try loading it
Akabei::Delta *delta = Akabei::Backend::instance()->loadDeltaFromFile(target);
if (delta) {
deltas.insert(delta->targetName(), delta);
qDebug() << "Added delta: " << delta->targetName() << delta->versionFrom() << delta->versionTo();
++addedDeltas;
}
qDebug() << "Added delta: " << tName << vfrom << vto;
++addedDeltas;
continue;
}
......@@ -252,7 +291,7 @@ void CreateDb::start()
// Done. Now we have to check for deltas and eventually create them.
// While we're at it, also populate the targets for the database insertion.
QList< Akabei::Package* > dbtargets;
QHash< QString, Akabei::Package* > dbtargets;
for (QHash< QString, QList < Akabei::Package* > >::const_iterator i = packages.constBegin(); i != packages.constEnd(); ++i) {
if (i.value().size() > 1) {
// Ok, let's actually order the list
......@@ -262,7 +301,7 @@ void CreateDb::start()
candidates = candidates.mid(candidates.count() - MAX_DELTAS);
}
// And add the very latest package to the DB
dbtargets.append(candidates.last());
dbtargets.insert(candidates.last()->name(), candidates.last());
// Cool. Now first of all, is there a margin for generating a delta?
bool create = false;
......@@ -288,14 +327,16 @@ void CreateDb::start()
(*k)->version().toByteArray() + ".delta";
// Is delta j -> k already done?
bool done = false;
QMultiHash< QString, QPair< QByteArray, QByteArray > >::const_iterator w = deltas.constFind(i.key());
QMultiHash< QString, Akabei::Delta* >::const_iterator w = deltas.constFind(i.key());
while (w != deltas.constEnd() && w.key() == i.key()) {
// Check versions
if (w.value().first == (*j)->version().toByteArray() &&
w.value().second == (*k)->version().toByteArray()) {
if (w.value()->versionFrom() == (*j)->version().toByteArray() &&
w.value()->versionTo() == (*k)->version().toByteArray()) {
// The delta is already in
done = true;
break;
}
++w;
}
if (done) {
// Skip it
......@@ -315,18 +356,33 @@ void CreateDb::start()
proc.start(command);
proc.waitForFinished(600000);
if (proc.exitCode() == 0) {
// Generate .DELTAINFO
QTemporaryFile deltainfo;
deltainfo.open();
QTextStream indelta(&deltainfo);
indelta << "# Generated by akabei-create-db" << endl;
indelta << "# " << QDateTime::currentDateTime().toString(Qt::SystemLocaleLongDate) << endl;
indelta << "pkgname = " << i.key() << endl;
indelta << "verfrom = " << (*j)->version().toByteArray() << endl;
indelta << "verto = " << (*k)->version().toByteArray() << endl;
// Ok, now compress the delta
out << "Done!" << endl;
out << '\t' << "Compressing delta... ";
out.flush();
QHash<QString,QString> files;
files.insert(".DELTAINFO", deltainfo.fileName());
files.insert(deltaName.split('/').last(), deltaName);
QString realDeltaName = deltaName;
deltaName += ".tar.xz";
if (!writeArchive(deltaName, files)) {
out << "Failed! Could not compress the delta. Skipping..." << endl;
QFile::remove(realDeltaName);
continue;
}
QFile::remove(realDeltaName);
out << "Done!" << endl;
// Done. Is the delta convenient?
......@@ -338,9 +394,20 @@ void CreateDb::start()
<< (deltaInfo.size() * 100) / (*j)->size() << "%, skipping..." << endl;
continue;
}
// Insert it!
deltas.insert(i.key(), qMakePair((*j)->version().toByteArray(), (*k)->version().toByteArray()));
out << "Done!" << endl;
// Ok, attempt to load the delta now
out << '\t' << "Validating delta... ";
out.flush();
Akabei::Delta *delta = Akabei::Backend::instance()->loadDeltaFromFile(deltaName);
if (delta) {
// Insert it!
deltas.insert(delta->targetName(), delta);
out << "Done!" << endl;
} else {
QFile::remove(deltaName);
out << "Failed!" << endl;
}
} else {
// Failed
out << "Failed!" << endl;
......@@ -350,7 +417,7 @@ void CreateDb::start()
}
} else {
// Simply add it to the list
dbtargets << i.value().first();
dbtargets.insert(i.value().first()->name(), i.value().first());
}
}
......@@ -373,18 +440,18 @@ void CreateDb::start()
out.flush();
int idx = 0;
// Now let's start adding the various targets
foreach (Akabei::Package *p, dbtargets) {
for (QHash< QString, Akabei::Package* >::const_iterator i = dbtargets.constBegin(); i != dbtargets.constEnd(); ++i) {
++idx;
// Prepare the insert query
int ret = Akabei::Helpers::QueryPerformer::insertPackage(db, p);
int ret = Akabei::Helpers::QueryPerformer::insertPackage(db, i.value());
if (ret != SQLITE_OK) {
out << "Failed! Could not perform the query. SQLite error code: " << ret << ". Skipping target." << endl;
++skipped;
continue;
}
if (p->hasScriptlet()) {
int ret = Akabei::Helpers::QueryPerformer::insertScriptlet(db, p);
if (i.value()->hasScriptlet()) {
int ret = Akabei::Helpers::QueryPerformer::insertScriptlet(db, i.value());
if (ret != SQLITE_OK) {
out << "Failed! Could not perform the query. SQLite error code: " << ret << ". Skipping target." << endl;
++skipped;
......@@ -393,7 +460,7 @@ void CreateDb::start()
}
// Prepare the insert query
ret = Akabei::Helpers::QueryPerformer::insertFiles(db, p, QString());
ret = Akabei::Helpers::QueryPerformer::insertFiles(db, i.value(), QString());
if (ret != SQLITE_OK) {
out << "Failed! Could not perform the query. SQLite error code: " << ret << ". Skipping target." << endl;
++skipped;
......@@ -407,8 +474,39 @@ void CreateDb::start()
}
// Add deltas!
{
// TODO
for (QHash< QString, Akabei::Delta* >::const_iterator i = deltas.constBegin(); i != deltas.constEnd(); ++i) {
QString sqlQuery = "INSERT INTO deltas ('Package', 'Filename', 'VersionFrom', 'VersionTo', 'MD5SUM') "
"VALUES (:Package, :Filename, :VersionFrom, :VersionTo, :MD5SUM)";
// Now prepare
sqlite3_stmt *stmt = 0;
int result = sqlite3_prepare16_v2(db, sqlQuery.constData(), (sqlQuery.size() + 1) * sizeof(QChar),
&stmt, 0);
switch (result) {
case SQLITE_OK:
// Everything went fine and the handle is ready
break;
default:
// Errors. Take action here
continue;
break;
}
// Now go for all the bindings
if (!performBind(stmt, ":VersionFrom", i.value()->versionFrom())) continue;
if (!performBind(stmt, ":VersionTo", i.value()->versionTo())) continue;
if (!performIntBind(stmt, ":Package", dbtargets[i.key()]->databaseId())) continue;
if (!performBind(stmt, ":Filename", i.value()->filename())) continue;
if (!performBind(stmt, ":MD5SUM", i.value()->md5sum())) continue;
// Ok, do the query
result = sqlite3_step(stmt);
if (result != SQLITE_DONE) {
// errors...
continue;
}
sqlite3_finalize(stmt);
}
out << endl;
......
......@@ -15,6 +15,7 @@
#include <QtCore/QStringList>
#include <QtCore/QTextStream>
#include <QtCore/QHash>
#include <sqlite3.h>
class CreateDb : public QObject
{
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment