backupservice.php

<?php
/**
 * ownCloud - nextbackup
 *
 * This file is licensed under the Affero General Public License version 3 or
 * later. See the COPYING file.
 *
 * @author Patrizio Bekerle <patrizio@bekerle.com>
 * @copyright Patrizio Bekerle 2015
 */
namespace OCA\NextBackup\Service;

use Exception;
use OCP\IDBConnection;
use OCP\ILogger;

class BackupService {

    private $appName;
    private $odb;
    private $configService;
    private $logger;
    private $logContext;
    private $userId;
    private $db;


    // how many backups do we want to keep in each interval
    private static $maxBackupTimestampsPerInterval = array(
        // for 24h, keep one backup every hour (a maximum of 24 backups are kept that have an interval of at least 1h)
        array('amount' => 24, 'interval' => 3600),
        // for 7d, keep one backup per day (a maximum of 7 backups are kept that have an interval of at least 1d)
        array('amount' => 7,  'interval' => 86400),
        // for 4w, keep one backup per week (a maximum of 4 backups are kept that have an interval of at least 1w)
        array('amount' => 4,  'interval' => 604800),
        // for 12m, keep one backup per 30d (a maximum of 12 backups are kept that have an interval of at least 30d)
        array('amount' => 12, 'interval' => 2592000),
        // next 2y, keep one backup per year (a maximum of 2 backups are kept that have an interval of at least 1y)
        array('amount' => 2,  'interval' => 31536000),
    );


    // the minimal interval for backups [s]
    const MIN_BACKUP_INTERVAL = 3600;


    /**
     * BackupService constructor
     *
     * @param $appName
     * @param \OC_DB $odb
     * @param ConfigService $configService
     * @param ILogger $logger
     * @param $userId
     * @param $db
     */
    public function __construct($appName, \OC_DB $odb, IDBConnection $db, ConfigService $configService, ILogger $logger, $userId){
        $this->appName = $appName;
        $this->odb = $odb;
        $this->configService = $configService;
        $this->logger = $logger;
        $this->logContext = ['app' => 'nextbackup'];
        $this->userId = $userId;
        $this->db = $db;
    }

    /**
     * Returns the id of the user or the name of the app if no user is present (for example in a cronjob)
     *
     * @return string
     */
    private function getCallerName()
    {
        return is_null( $this->userId ) ? $this->appName : "user " . $this->userId;
    }

    /**
     * Returns the complete serialized dump of a table without field names
     *
     * @param string $table
     * @return array
     */
    private function getTableSerializedDataDump( $table )
    {
        $sql = "SELECT * FROM `$table`";
        $query = $this->db->executeQuery( $sql );
        $result = $query->fetchAll();

        return serialize( array_map( function( $array ) {
            // we want no field names, just the values, to safe space
            return array_values( $array );
        }, $result ) );
    }

    /**
     * Runs a backup of all tables to sql files
     *
     * @throws Exception
     * @return int timestamp of backup
     */
    public function createDBBackup()
    {
        $timestamp = time();

        try
        {
            $backupDir = $this->configService->getBackupBaseDirectory(). "/$timestamp";

            // create new backup folder if it not exists
            if ( !file_exists( $backupDir ) )
            {
                if ( !mkdir( $backupDir ) )
                {
                    throw new Exception( "Cannot create backup dir: $backupDir" );
                }
            }

            $structureFile = "$backupDir/structure.xml";

            // get the db structure
            // Warning: does not seem to support `longblob`!
            // @see: https://github.com/owncloud/core/issues/25316
            if ( !$this->odb->getDbStructure( $structureFile ) )
            {
                throw new Exception( "Cannot create db structure in file: $structureFile" );
            }

            // create a xml object from db structure
            $loadEntities = libxml_disable_entity_loader(false);
            $xml = simplexml_load_file( $structureFile );
            libxml_disable_entity_loader($loadEntities);

            $charset = (string) $xml->charset;

            /** @var \SimpleXMLElement $child */
            foreach ($xml->children() as $child)
            {
                // skip everything but tables
                if ( $child->getName() !== "table" )
                {
                    continue;
                }

                // find the table name
                $tableName = (string) $child->name;

                if ( $tableName === "" )
                {
                    throw new Exception( "No table name was set!" );
                }

                // build a structure xml for a single table
                $xmlDump = "<database><name>*dbname*</name><create>true</create><overwrite>false</overwrite><charset>$charset</charset>" . $child->asXML() . "</database>";

                // get name of table directory
                $tableDir = "$backupDir/$tableName";

                // create table directory if it does not exist
                if ( !file_exists( $tableDir ) )
                {
                    if ( !mkdir( $tableDir ) )
                    {
                        throw new Exception( "Cannot create table dir: $tableDir" );
                    }
                }

                // write structure to table structure file
                $tableStructureFile = "$tableDir/structure.xml";
                file_put_contents( $tableStructureFile, $xmlDump );

                // get a serialized dump of the table
                $tableDump = $this->getTableSerializedDataDump( $tableName );

                // write dump to table data file (compressed if possible)
                $tableDataFile = "$tableDir/data.dump";
                file_put_contents( $tableDataFile, $this->tryToCompressString( $tableDump ) );
            }

            $this->logger->notice( $this->getCallerName() . " created a backup to '$backupDir'", $this->logContext );

            return $timestamp;
        }
        catch ( Exception $e )
        {
            $this->logger->error( $this->getCallerName() . " thew an exception: " . $e->getMessage(), $this->logContext );
            throw( $e );
        }
    }

    /**
     * Fetches all backup timestamp
     *
     * @return array
     * @throws Exception
     */
    public function fetchBackupTimestamps()
    {
        $backupBaseDir = $this->configService->getBackupBaseDirectory();
        $timestampList = array();

        $fileList = scandir( $backupBaseDir, 1 );
        foreach ( $fileList as $file )
        {
            // skip "." and ".."
            if ( $file === "." || $file === ".." )
            {
                continue;
            }

            $fullFileName = "$backupBaseDir/$file";

            // only add directories to the list
            if ( is_dir( $fullFileName ) && is_readable( $fullFileName ) )
            {
                $timestampList[] = (int) $file;
            }
        }

        rsort( $timestampList, SORT_NUMERIC );
        return $timestampList;
    }

    /**
     * Fetches all backup timestamps as hash with formatted date strings
     *
     * @return array
     */
    public function fetchFormattedBackupTimestampHash()
    {
        $timestampList = $this->fetchBackupTimestamps();

        $dateTimeFormatter = \OC::$server->query('DateTimeFormatter');
        $dateHash = [];
        foreach( $timestampList as $timestamp )
        {
            $dateHash[$timestamp] = $dateTimeFormatter->formatDateTime( $timestamp );
        }

        krsort( $dateHash, SORT_NUMERIC );
        return $dateHash;
    }

    /**
     * Fetches all table names of the backup with a certain timestamp
     *
     * @param int $timestamp
     * @return array|false
     * @throws Exception
     */
    public function fetchTablesFromBackupTimestamp( $timestamp )
    {
        $timestamp = (int) $timestamp;

        if ( $timestamp === 0 )
        {
            return false;
        }

        $backupDir = $this->configService->getBackupBaseDirectory() . "/$timestamp";
        $tableList = [];

        $fileList = scandir( $backupDir );
        foreach ( $fileList as $file )
        {
            // skip "." and ".."
            if ( $file === "." || $file === ".." )
            {
                continue;
            }

            $fullFileName = "$backupDir/$file";

            // only add directories to the list
            if ( is_dir( $fullFileName ) && is_readable( $fullFileName ) )
            {
                $tableList[] = $file;
            }
        }

        return $tableList;
    }

    /**
     * Restores a table for a timestamp
     *
     * @param int $timestamp
     * @param string $table
     * @return bool
     * @throws Exception
     */
    public function doRestoreTable( $timestamp, $table )
    {
        $timestamp = (int) $timestamp;

        if ( $timestamp === 0 )
        {
            return false;
        }

        if ( preg_match( '/[\.\/]/', $table ) )
        {
            throw new Exception( "Invalid table name: $table" );
        }

        // get the table structure file name
        $structureFile = $this->configService->getBackupBaseDirectory() . "/$timestamp/$table/structure.xml";

        $this->db->beginTransaction();

        if ( !is_file( $structureFile ) || !is_readable( $structureFile ) )
        {
            throw new Exception( "Cannot read table structure file: $structureFile" );
        }

        // drops the table
        $this->dropTable( $table );

        // update the table structure
        if ( !$this->odb->updateDbFromStructure( $structureFile ) )
        {
            throw new Exception( "Cannot restore table structure from file: $structureFile" );
        }

        // get the data dump file name
        $dataDumpFile = $this->configService->getBackupBaseDirectory() . "/$timestamp/$table/data.dump";

        if ( !is_file( $dataDumpFile ) || !is_readable( $dataDumpFile ) )
        {
            throw new Exception( "Cannot read table data dump file: $dataDumpFile" );
        }

        // try to get the data dump
        $dataDump = unserialize( $this->tryToUncompressString( file_get_contents( $dataDumpFile ) ) );
        if ( !is_array( $dataDump ) )
        {
            throw new Exception( "Data dump is no array in file: $dataDumpFile" );
        }

        // generate the field name list from the table structure file
        $fieldList = $this->getFieldListFromTableStructureFile( $structureFile );

        // insert all the data
        foreach( $dataDump as $dataLine )
        {
            $dataHash = [];

            // generate the data hash
            foreach ( $fieldList as $key => $fieldName )
            {
                // we want to add the field names again, that we left out to save space
                $dataHash[$fieldName] = $dataLine[$key];
            }

            try
            {
                // insert the data into table
                $this->db->insertIfNotExist( $table, $dataHash );
            }
            catch( \Exception $e )
            {
                throw new Exception( "Inserting data row failed: " . $e->getMessage() );
            }
        }
        $this->db->commit();

        $this->logger->notice( $this->getCallerName() . " restored table '$table' from backup $timestamp", $this->logContext );

        return true;
    }

    /**
     * Generates a field name list from a table structure file
     *
     * @param string $tableStructureFile
     * @return array
     */
    private function getFieldListFromTableStructureFile( $tableStructureFile )
    {
        // create a xml object from table structure
        $loadEntities = libxml_disable_entity_loader(false);
        $xml = simplexml_load_file( $tableStructureFile );
        libxml_disable_entity_loader($loadEntities);

        $fieldList = [];

        /** @var \SimpleXMLElement $tableDeclaration */
        $tableDeclaration = $xml->table->declaration;

        /** @var \SimpleXMLElement $child */
        foreach( $tableDeclaration->children() as $child )
        {
            // skip everything but fields
            if ( $child->getName() !== "field" )
            {
                continue;
            }

            $fieldName = (string) $child->name;
            $fieldList[] = $fieldName;
        }

        return $fieldList;
    }

    /**
     * Drops a table
     *
     * @param $table
     * @return mixed
     * @throws Exception
     */
    private function dropTable( $table )
    {
        // remove the prefix from the table name
        $filterExpression = '/^' . preg_quote( $this->configService->getSystemValue( 'dbtableprefix', 'oc_' ) ) . '/';
        $tableNoPrefix = preg_replace( $filterExpression, "", $table );

        if ( $tableNoPrefix === "" )
        {
            throw new Exception( "Cannot remove prefix from table name: $table" );
        }

        return $this->db->dropTable( $tableNoPrefix );
    }

    /**
     * Restores a list of tables for a timestamp
     *
     * @param int $timestamp
     * @param array $tables
     * @return bool
     * @throws Exception
     */
    public function doRestoreTables( $timestamp, array $tables )
    {
        $timestamp = (int) $timestamp;

        if ( $timestamp === 0 )
        {
            return false;
        }

        try
        {
            // enabled maintenance mode
            $this->configService->setSystemValue('maintenance', true);

            $this->db->beginTransaction();

            foreach ( $tables as $table )
            {
                // restore a table
                $this->doRestoreTable( $timestamp, $table );
            }

            $this->db->commit();
        }
        catch( \Exception $e )
        {
            // do a rollback
            $this->db->rollBack();

            // disable maintenance mode
            $this->configService->setSystemValue('maintenance', false);

            throw $e;
        }

        // disable maintenance mode
        $this->configService->setSystemValue('maintenance', false);

        return true;
    }

    /**
     * Fetches the timestamp of the last backup
     *
     * @return int|bool
     */
    public function fetchLastBackupTimestamp()
    {
        // fetch all backup timestamps
        $backupTimestamps = $this->fetchBackupTimestamps();

        // sort them descending
        rsort( $backupTimestamps );

        return isset( $backupTimestamps[0] ) ? (int) $backupTimestamps[0] : false;
    }

    /**
     * Checks if we need a new backup
     *
     * @return bool
     */
    public function needNewBackup()
    {
        return ( (int) $this->fetchLastBackupTimestamp() ) < ( time() - self::MIN_BACKUP_INTERVAL );
    }

    /**
     * Expires all old backups
     *
     * @return array timestamp list of removed backups
     */
    public function expireOldBackups()
    {
        // fetch all backup timestamps
        $backupTimestamps = $this->fetchBackupTimestamps();

        // get the list of backup timestamps we want to expire
        $removeTimestampList = self::getAutoExpireList( $backupTimestamps );
        $removedTimestampList = [];

        // expire old backups
        foreach ( $removeTimestampList as $timestamp )
        {
            // remove backup
            if ( $this->removeBackup( $timestamp ) )
            {
                $removedTimestampList[] = $timestamp;
            }
        }

        return $removedTimestampList;
    }

    /**
     * Returns a list of timestamp meeting a certain interval
     *
     * @param integer[] $timestamps
     * @param integer $interval
     * @param integer|null $keepAmount
     * @return integer[]
     */
    protected static function findIntervalTimestamps( array $timestamps, $interval, $keepAmount = null )
    {
        if ( count( $timestamps ) === 0 ) {
            return [];
        }

        // descending order is crucial here
        rsort( $timestamps, SORT_NUMERIC );

        // keep all if not set
        if ( is_null( $keepAmount ) ) {
            $keepAmount = count( $timestamps );
        }

        $resultList = [];
        $lastTimestamp = $timestamps[0] + $interval;
        $count = 0;

        foreach ( $timestamps as $timestamp )
        {
            // gather timestamps in our interval range
            if ( $timestamp <= ( $lastTimestamp - $interval ) )
            {
                $resultList[] = $timestamp;
                $lastTimestamp = $timestamp;
                $count++;

                // check if we have enough timestamps
                if ( $count >= $keepAmount ) {
                    break;
                }
            }
        }

        // if we don't have enough timestamps to keep we want to keep at least the last (lowest) timestamp to build up the list after future backups
        if ( $count < $keepAmount )
        {
            $timestamp = end( $timestamps );
            if ( !in_array( $timestamp, $resultList ) )
            {
                $resultList[] = $timestamp;
            }
        }

        return $resultList;
    }

    /**
     * Returns a list of timestamp meeting a certain interval, but from oldest to newest
     * This is a fallback for @see BackupService::findIntervalTimestamps
     *
     * @param integer[] $timestamps
     * @param integer $interval
     * @param integer|null $keepAmount
     * @return integer[]
     */
    protected static function findIntervalTimestampsFallback( array $timestamps, $interval, $keepAmount = null )
    {
        if ( count( $timestamps ) === 0 ) {
            return [];
        }

        // ascending order is crucial here
        sort( $timestamps, SORT_NUMERIC );

        // keep all if not set
        if ( is_null( $keepAmount ) ) {
            $keepAmount = count( $timestamps );
        }

        $resultList = [];
        $lastTimestamp = $timestamps[0] - $interval;
        $count = 0;

        foreach ( $timestamps as $timestamp )
        {
            // gather timestamps in our interval range
            if ( $timestamp >= ( $lastTimestamp + $interval ) )
            {
                $resultList[] = $timestamp;
                $lastTimestamp = $timestamp;
                $count++;

                // check if we have enough timestamps
                if ( $count >= $keepAmount ) {
                    break;
                }
            }
        }

        return $resultList;
    }

    /**
     * Returns a list of backup timestamps we want to expire
     *
     * @param integer[] $timestamps list of timestamps
     * @return integer[] containing the list of to be deleted timestamps
     */
    protected static function getAutoExpireList( array $timestamps )
    {
        if ( count( $timestamps ) === 0 ) {
            return [];
        }

        $timestampsToKeep = [];
        // check all intervals
        foreach( self::$maxBackupTimestampsPerInterval as $intervalData )
        {
            $keepAmount = (int) $intervalData["amount"];
            $interval = (int) $intervalData["interval"];

            // get all timestamps we need for this interval
            $foundTimestamps = self::findIntervalTimestamps( $timestamps, $interval, $keepAmount );

            // we got too few timestamps lets try it the other way around to make sure we keep enough over the time
            if ( count( $foundTimestamps ) < $keepAmount )
            {
                $moreFoundTimestamps = self::findIntervalTimestampsFallback( $timestamps, $interval, $keepAmount );
                $foundTimestamps = array_merge( $foundTimestamps, $moreFoundTimestamps );
            }

            // merge the found timestamps with the current timestamps we need to keep
            $timestampsToKeep = array_merge( $timestampsToKeep, $foundTimestamps );
        }

        // get the timestamps we want to expire
        $timestampsToDelete = array_diff( $timestamps, $timestampsToKeep );

        return $timestampsToDelete;
    }


    /**
     * Removes a backup
     *
     * @param int $timestamp
     * @return bool
     * @throws Exception
     */
    public function removeBackup( $timestamp )
    {
        $timestamp = (int) $timestamp;

        if ( $timestamp === 0 )
        {
            return false;
        }

        $backupDir = $this->configService->getBackupBaseDirectory() . "/$timestamp";
        $success = false;

        if ( is_dir( $backupDir ) && is_writeable( $backupDir ) )
        {
            $success = $this->recursivelyRemoveDir( $backupDir );
        }

        if ( $success )
        {
            $this->logger->notice( $this->getCallerName() . " removed backup $timestamp", $this->logContext );
        }
        else
        {
            $this->logger->error( $this->getCallerName() . " could not remove backup directory '$backupDir'!", $this->logContext );
        }

        return $success;
    }

    /**
     * Attempts to compress a string
     *
     * @param $text
     * @return string
     */
    private function tryToCompressString( $text )
    {
        if ( function_exists( "gzencode" ) )
        {
            $compressedText = gzencode( $text );
            if ( $compressedText !== false )
            {
                return $compressedText;
            }
        }

        return $text;
    }

    /**
     * Attempts to uncompress a string
     *
     * @param $compressedText
     * @return string
     */
    private function tryToUncompressString( $compressedText )
    {
        if ( function_exists( "gzdecode" ) )
        {
            $text = gzdecode( $compressedText );
            if ( $text !== false )
            {
                return $text;
            }
        }

        return $compressedText;
    }

    /**
     * Recursively removes a directory
     *
     * @param string $dir
     * @return bool
     */
    private function recursivelyRemoveDir( $dir )
    {
        $success = false;
        if (is_dir($dir)) {
            $objects = scandir($dir);
            foreach ($objects as $object) {
                if ($object !== "." && $object !== "..") {
                    if (filetype($dir."/".$object) === "dir") $this->recursivelyRemoveDir($dir."/".$object); else unlink($dir."/".$object);
                }
            }
            reset($objects);
            $success = rmdir($dir);
        }

        return $success;
    }
}