# -*- coding: utf-8 -*-
"""

    @author: Fabio Erculiani <lxnay@sabayon.org>
    @contact: lxnay@sabayon.org
    @copyright: Fabio Erculiani
    @license: GPL-2

    B{Entropy Package Manager Client EntropyRepository plugin code}.

"""
import codecs
import errno
import os
import shutil
import subprocess
import sys
import threading
import time

from entropy.const import const_debug_write, const_setup_perms, etpConst, \
    const_set_nice_level, const_setup_file, const_convert_to_unicode, \
    const_debug_enabled, const_mkdtemp, const_mkstemp, const_file_readable, \
    const_file_writable
from entropy.output import blue, darkred, red, darkgreen, purple, teal, brown, \
    bold, TextInterface
from entropy.dump import dumpobj, loadobj
from entropy.cache import EntropyCacher
from entropy.db import EntropyRepository
from entropy.exceptions import RepositoryError, SystemDatabaseError, \
    PermissionDenied
from entropy.security import Repository as RepositorySecurity
from entropy.misc import TimeScheduled, ParallelTask
from entropy.fetchers import UrlFetcher
from entropy.i18n import _
from entropy.db.skel import EntropyRepositoryPlugin, EntropyRepositoryBase
from entropy.db.exceptions import IntegrityError, OperationalError, Error, \
    DatabaseError
from entropy.core.settings.base import SystemSettings
from entropy.services.client import WebService
from entropy.client.services.interfaces import RepositoryWebService, \
    RepositoryWebServiceFactory

import entropy.dep
import entropy.tools

__all__ = ["CachedRepository", "ClientEntropyRepositoryPlugin",
    "InstalledPackagesRepository", "AvailablePackagesRepository",
    "GenericRepository"]

class ClientEntropyRepositoryPlugin(EntropyRepositoryPlugin):

    def __init__(self, client_interface, metadata = None):
        """
        Entropy client-side repository EntropyRepository Plugin class.
        This class will be instantiated and automatically added to
        EntropyRepository instances generated by Entropy Client.

        @param client_interface: Entropy Client interface instance
        @type client_interface: entropy.client.interfaces.Client class
        @param metadata: any dict form metadata map (key => value)
        @type metadata: dict
        """
        EntropyRepositoryPlugin.__init__(self)
        self._client = client_interface
        if metadata is None:
            self._metadata = {}
        else:
            self._metadata = metadata

    def get_id(self):
        return "__client__"

    def get_metadata(self):
        return self._metadata

    def add_plugin_hook(self, entropy_repository_instance):
        const_debug_write(__name__,
            "ClientEntropyRepositoryPlugin: calling add_plugin_hook => %s" % (
                self,)
            )

        out_intf = self._metadata.get('output_interface')
        if out_intf is not None:
            entropy_repository_instance.output = out_intf.output
            entropy_repository_instance.ask_question = out_intf.ask_question

        return 0

class CachedRepository(EntropyRepository):
    """
    This kind of repository cannot have close() called directly, without
    a valid token passed. This is because the class object is cached somewhere
    and calling close() would turn into a software bug.
    """
    def setCloseToken(self, token):
        """
        Set a token that can be used to validate close() calls. Calling
        close() on these repos is prohibited and considered a software bug.
        Only Entropy Client should be able to close them.
        """
        self._close_token = token

    def close(self, safe=False, _token = None):
        """
        Reimplemented from EntropyRepository
        """
        close_token = getattr(self, "_close_token", None)
        if close_token is not None:
            if (_token is None) or (_token != close_token):
                raise PermissionDenied(
                    "cannot close this repository directly. Software bug!")
        return EntropyRepository.close(self, safe=safe)


class InstalledPackagesRepository(CachedRepository):
    """
    This class represents the installed packages repository and is a direct
    subclass of EntropyRepository.
    """

    # Name of the repository
    NAME = "__system__"

    def __init__(self, *args, **kwargs):
        # force our own name, always.
        kwargs = kwargs.copy()
        kwargs['name'] = self.NAME
        super(InstalledPackagesRepository, self).__init__(
            *args, **kwargs)

        # ensure proper repository file permissions
        if entropy.tools.is_root() and os.path.isfile(self._db):
            const_setup_file(self._db, etpConst['entropygid'], 0o644,
                uid = etpConst['uid'])

    def handlePackage(self, pkg_data, revision = None,
                      formattedContent = False):
        """
        Reimplemented from EntropyRepository.
        """
        removelist = self.getPackagesToRemove(
            pkg_data['name'], pkg_data['category'],
            pkg_data['slot'], pkg_data['injected']
        )
        for r_package_id in removelist:
            self.removePackage(r_package_id)
        return self.addPackage(pkg_data, revision = revision,
            formatted_content = formattedContent)


class AvailablePackagesRepositoryUpdater(object):

    """
    AvailablePackagesRepository update logic class.
    The required logic for updating a repository is stored here.
    """
    WEBSERV_CACHE_ID = 'webserv_repo/segment_'

    FETCH_ERRORS = (
        UrlFetcher.GENERIC_FETCH_WARN,
        UrlFetcher.TIMEOUT_FETCH_ERROR,
        UrlFetcher.GENERIC_FETCH_ERROR)

    def __init__(self, entropy_client, repository_id, force, gpg):
        self.__force = force
        self.__big_sock_timeout = 20
        self._repository_id = repository_id
        self._cacher = EntropyCacher()
        self._entropy = entropy_client
        self._settings = SystemSettings()
        self._gpg_feature = gpg
        self._supported_apis = etpConst['supportedapis']
        self._supported_download_items = (
            "db", "dbck", "dblight", "ck", "cklight", "compck",
            "lock", "dbdump", "dbdumplight", "dbdumplightck", "dbdumpck",
            "meta_file", "meta_file_gpg", "notice_board"
        )
        self._developer_repo = \
            self._settings['repositories']['developer_repo']
        self._differential_update = \
            self._settings['repositories']['differential_update']
        if self._developer_repo:
            const_debug_write(__name__,
                "__init__: developer repo mode enabled")
        self.__webservices = None
        self.__webservice = None
        self.__repo_eapi = None

        avail_data = self._settings['repositories']['available']
        if self._repository_id not in avail_data:
            raise KeyError("Repository not available")

    @property
    def _repo_eapi(self):
        if self.__repo_eapi is None:
            self.__repo_eapi = self.__get_repo_eapi()
        return self.__repo_eapi

    @_repo_eapi.setter
    def _repo_eapi(self, value):
        self.__repo_eapi = value

    @property
    def _webservices(self):
        if self.__webservices is None:
            if hasattr(self._entropy, "RepositoryWebServices"):
                self.__webservices = self._entropy.RepositoryWebServices()
            else:
                # in case self._entropy is a simple TextInterface()
                # like how it's called in remote_revision().
                self.__webservices = RepositoryWebServiceFactory(self._entropy)
                # cross fingers!

        return self.__webservices

    @property
    def _webservice(self):
        if self.__webservice is None:
            self.__webservice = self._webservices.new(self._repository_id)
            self.__webservice._set_timeout(self.__big_sock_timeout)
        return self.__webservice

    def __get_webserv_repository_metadata(self):
        try:
            data = self._webservice.get_repository_metadata()
        except WebService.WebServiceException as err:
            const_debug_write(__name__,
                "__get_webserv_repository_metadata: error: %s" % (err,))
            data = {}
        return data

    def __get_webserv_repository_revision(self):
        try:
            revision = self._webservice.get_revision()
        except WebService.WebServiceException as err:
            const_debug_write(__name__,
                "__get_webserv_repository_revision: error: %s" % (err,))
            revision = None
        return revision

    def __check_webserv_availability(self):
        try:
            webserv = self._webservices.new(self._repository_id)
        except WebService.UnsupportedService:
            return False

        try:
            available = webserv.update_service_available(cache = False)
        except WebService.WebServiceException:
            available = False
        return available

    def __get_webserv_local_database(self):

        avail_data = self._settings['repositories']['available']
        repo_data = avail_data[self._repository_id]

        dbfile = os.path.join(repo_data['dbpath'],
            etpConst['etpdatabasefile'])
        dbconn = None
        try:
            dbconn = self._entropy.open_generic_repository(dbfile,
                xcache = False, indexing_override = False)
            dbconn.validate()
        except (OperationalError, IntegrityError, SystemDatabaseError,
            IOError, OSError,):
            dbconn = None
        return dbconn

    def __get_webserv_database_differences(self, webserv, package_ids):

        try:
            remote_package_ids = webserv.get_package_ids()
        except WebService.WebServiceException as err:
            const_debug_write(__name__,
                "__get_webserv_database_differences: error: %s" % (err,))
            return None, None

        added = [x for x in remote_package_ids if x not in package_ids]
        removed = [x for x in package_ids if x not in remote_package_ids]
        # sort, to increase determinism
        added.sort()
        return added, removed

    def __eapi1_eapi2_databases_alignment(self, dbfile, dbfile_old):

        dbconn = self._entropy.open_generic_repository(dbfile,
            xcache = False, indexing_override = False)
        old_dbconn = self._entropy.open_generic_repository(dbfile_old,
            xcache = False, indexing_override = False)
        upd_rc = 0
        try:
            upd_rc = old_dbconn.alignDatabases(dbconn, output_header = "\t")
        except (OperationalError, IntegrityError, DatabaseError,):
            pass
        old_dbconn.close()
        dbconn.close()
        if upd_rc > 0:
            # -1 means no changes, == force used
            # 0 means too much hassle
            os.rename(dbfile_old, dbfile)
        return upd_rc

    def __eapi2_inject_downloaded_dump(self, dumpfile, dbfile, cmethod):

        # load the dump into database
        mytxt = "%s %s, %s %s" % (
            red(_("Injecting downloaded dump")),
            darkgreen(etpConst['etpdatabasedumplight']),
            red(_("please wait")),
            red("..."),
        )
        self._entropy.output(
            mytxt,
            importance = 0,
            level = "info",
            header = "\t"
        )
        dbconn = self._entropy.open_generic_repository(dbfile,
            xcache = False, indexing_override = False)
        rc = dbconn.importRepository(dumpfile, dbfile)
        dbconn.close()
        return rc

    def __get_repo_eapi(self):

        eapi_env = os.getenv("FORCE_EAPI")
        sqlite3_access = os.path.lexists("/usr/bin/sqlite3")
        sqlite3_rc = subprocess.call("/usr/bin/sqlite3 -version > /dev/null",
            shell = True)
        try:
            eapi_env_clear = int(eapi_env)
            if eapi_env_clear not in self._supported_apis:
                raise ValueError()
        except (ValueError, TypeError,):
            eapi_env_clear = None

        repo_eapi = 2
        eapi_avail = self.__check_webserv_availability()
        if eapi_avail:
            repo_eapi = 3
        else:
            if not sqlite3_access or entropy.tools.islive():
                repo_eapi = 1
            elif sqlite3_rc != 0:
                repo_eapi = 1

        # if differential update is disabled and FORCE_EAPI is not overriding
        # we cannot use EAPI=3
        if (eapi_env_clear is None) and (not self._differential_update) and \
            (repo_eapi == 3):
            const_debug_write(__name__,
                "__get_repo_eapi: differential update is disabled !")
            repo_eapi -= 1

        # check EAPI
        if eapi_env_clear is not None:
            repo_eapi = eapi_env_clear

            # FORCE_EAPI is triggered, disable
            # developer_repo mode
            self._developer_repo = False
            const_debug_write(__name__,
                "__get_repo_eapi: developer repo mode disabled FORCE_EAPI")

        elif repo_eapi > 1 and self._developer_repo:
            # enforce EAPI=1
            repo_eapi = 1

        const_debug_write(__name__,
            "__get_repo_eapi: final eapi set to %s" % (repo_eapi,))

        return repo_eapi

    def _is_repository_updatable(self, revision):
        """
        Given a remote repository revision, return whether
        the repository is updatable or not.
        """
        if self.__force:
            return True

        local = AvailablePackagesRepository.revision(
            self._repository_id)
        return local != revision

    def _select_database_mirror(self):
        """
        Verify that the database mirror URL is available and working.
        If this is not the case, fallback to the first available
        and working packages mirror URL.
        """
        repos_data = self._settings['repositories']
        avail_data = repos_data['available']
        repo_data = avail_data[self._repository_id]
        database_uris = repo_data['databases']

        ws_revision = self._remote_webservice_revision()

        # Setup the repository uri
        revision = None
        uri = None
        cformat = None
        for counter, uri_meta in enumerate(database_uris, 1):

            mytxt = "%s: %s [%s]" % (
                red(_("Scanning URL")),
                teal(uri_meta['uri']),
                brown(uri_meta['dbcformat']),)
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "info",
                header = blue("  # "),
                back = True,
                count = (counter, len(database_uris))
            )

            repo_uri = uri_meta['uri']
            uri_revision = self._remote_revision(repo_uri)

            if uri_revision != -1:

                mytxt = "%s: %s [%s]" % (
                    darkgreen(_("Selected URL")),
                    teal(uri_meta['uri']),
                    brown(uri_meta['dbcformat']),)
                self._entropy.output(
                    mytxt,
                    importance = 1,
                    level = "info",
                    header = blue("  # ")
                )

                revision = uri_revision
                uri = repo_uri
                cformat = uri_meta['dbcformat']
                break

        if uri is not None:
            if ws_revision is not None:
                return ws_revision, uri, cformat
            elif revision is not None:
                return revision, uri, cformat

        self._entropy.output(
            "%s: %s" % (
                darkred(_("Attention")),
                brown(_("repository is not available at the database URLs")),
                ),
            importance = 1, level = "warning", header = "\t",
        )
        self._entropy.output(
            "%s" % (
                purple(
                    _("Looking for an alternate route using package mirrors")
                    ),),
            importance = 1, level = "warning", header = "\t",
        )

        package_uris = repo_data['plain_packages']
        default_cformat = etpConst['etpdatabasefileformat']
        for package_uri in package_uris:
            self._entropy.output(
                "%s:" % (brown(_("Checking repository URL")),),
                importance = 1, level = "warning", header = "\t",
                )
            self._entropy.output(
                "%s" % (package_uri,),
                importance = 1, level = "warning", header = "\t",
                )

            url = entropy.tools.expand_plain_database_mirror(
                package_uri, repos_data['product'],
                self._repository_id,
                repos_data['branch'])
            if url == uri:
                # skip same URL
                continue
            revision = self._remote_revision(url)
            if revision != -1:
                # found
                self._entropy.output(
                    "%s:" % (brown(_("Found repository at URL")),),
                    importance = 1, level = "warning", header = "\t",
                    )
                self._entropy.output(
                    "%s" % (url,),
                    importance = 1, level = "warning", header = "\t",
                    )
                return revision, url, default_cformat

        self._entropy.output(
            "%s" % (
                purple(
                    _("Unable to find alternate repository mirrors. Sorry.")
                    ),),
            importance = 1, level = "warning", header = "\t",
        )
        return None

    def _show_repository_information(self):
        """
        Show the repository information to the user.
        """
        avail_data = self._settings['repositories']['available']
        repo_data = avail_data[self._repository_id]

        self._entropy.output(
            bold("%s") % ( repo_data['description'] ),
            importance = 2,
            level = "info",
            header = blue("  # ")
        )

        for uri_meta in repo_data['databases']:
            mytxt = "%s: %s [%s]" % (
                red(_("Repository URL")),
                darkgreen(uri_meta['uri']),
                brown(uri_meta['dbcformat']),)
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "info",
                header = blue("  # ")
            )

        mytxt = "%s: %s" % (red(_("Repository local path")),
            darkgreen(repo_data['dbpath']),)
        self._entropy.output(
            mytxt,
            importance = 0,
            level = "info",
            header = blue("  # ")
        )
        mytxt = "%s: %s" % (red(_("Repository API")),
            darkgreen(str(self._repo_eapi)),)
        self._entropy.output(
            mytxt,
            importance = 0,
            level = "info",
            header = blue("  # ")
        )

    def __append_gpg_signature_to_path(self, path):
        return path + etpConst['etpgpgextension']

    def _ensure_repository_path(self):
        """
        Make sure that the local repository directory has valid
        permissions and ownership.
        """
        avail_data = self._settings['repositories']['available']
        repo_data = avail_data[self._repository_id]

        # create dir if it doesn't exist
        if not os.path.isdir(repo_data['dbpath']):
            os.makedirs(repo_data['dbpath'], 0o755)

        try:
            items = os.listdir(etpConst['etpdatabaseclientdir'])
        except OSError:
            items = []

        # we cannot operate dir wide (etpdatabaseclientdir) because
        # there are lock files in there and we should not touch them
        for item in items:
            repo_dir_path = os.path.join(
                etpConst['etpdatabaseclientdir'],
                item)
            if not os.path.isdir(repo_dir_path):
                continue
            const_setup_perms(
                repo_dir_path,
                etpConst['entropygid'],
                f_perms = 0o644)

    def __remove_repository_files(self):
        sys_set = self._settings
        avail_data = sys_set['repositories']['available']
        repo_dbpath = avail_data[self._repository_id]['dbpath']
        shutil.rmtree(repo_dbpath, True)

    def __database_download(self, uri, cmethod):

        mytxt = "%s ..." % (red(_("Downloading repository")),)
        self._entropy.output(
            mytxt,
            importance = 1,
            level = "info",
            header = "\t"
        )

        downloaded_item = None
        down_status = False
        sig_status = False
        if self._repo_eapi == 2:

            down_item = "dbdumplight"

            down_status = self._download_item(
                uri, down_item, cmethod,
                disallow_redirect = True)
            if down_status:
                # get GPG file if available
                sig_status = self._download_item(
                    uri, down_item, cmethod,
                    disallow_redirect = True,
                    get_signature = True)

            downloaded_item = down_item

        if not down_status: # fallback to old db

            self._repo_eapi = 1
            down_item = "dblight"
            if self._developer_repo:
                # if developer repo mode is enabled, fetch full-blown db
                down_item = "db"
                const_debug_write(__name__,
                    "__handle_database_download: developer repo mode enabled")

            down_status = self._download_item(
                uri, down_item, cmethod,
                disallow_redirect = True)
            if down_status:
                sig_status = self._download_item(
                    uri, down_item, cmethod,
                    disallow_redirect = True,
                    get_signature = True)

            downloaded_item = down_item

        if not down_status:
            mytxt = "%s: %s." % (bold(_("Attention")),
                red(_("unable to download the repository")),)
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = "\t"
            )

        return down_status, sig_status, downloaded_item

    def __database_checksum_download(self, uri, cmethod):

        downitem = 'cklight'
        if self._developer_repo:
            downitem = 'dbck'
        if self._repo_eapi == 2: # EAPI = 2
            downitem = 'dbdumplightck'

        garbage_url, hashfile = self._construct_paths(
            uri, downitem, cmethod)
        mytxt = "%s %s %s" % (
            red(_("Downloading checksum")),
            darkgreen(os.path.basename(hashfile)),
            red("..."),
        )
        # download checksum
        self._entropy.output(
            mytxt,
            importance = 0,
            level = "info",
            header = "\t"
        )

        db_down_status = self._download_item(
            uri, downitem, cmethod,
            disallow_redirect = True)

        if not db_down_status and (downitem not in ('cklight', 'dbck',)):
            # fallback to old method
            retryitem = 'cklight'
            if self._developer_repo:
                retryitem = 'dbck'
            db_down_status = self._download_item(
                uri, retryitem, cmethod,
                disallow_redirect = True)

        if not db_down_status:
            mytxt = "%s %s !" % (
                red(_("Cannot fetch checksum")),
                red(_("Cannot verify repository integrity")),
            )
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = "\t"
            )
        return db_down_status

    def __verify_file_checksum(self, file_path, md5_checksum_path):
        enc = etpConst['conf_encoding']
        with codecs.open(md5_checksum_path, "r", encoding=enc) as ck_f:
            md5hash = ck_f.readline().strip()
            if not md5hash: # invalid !! => [] would cause IndexError
                return False
            md5hash = md5hash.split()[0]
        return entropy.tools.compare_md5(file_path, md5hash)

    def __verify_database_checksum(self, uri, cmethod = None):

        sys_settings_repos = self._settings['repositories']
        avail_config = sys_settings_repos['available'][self._repository_id]

        sep = os.path.sep
        if self._repo_eapi == 1:
            if self._developer_repo:
                remote_gb, dbfile = self._construct_paths(
                    uri, "db", cmethod)
                remote_gb, md5file = self._construct_paths(
                    uri, "dbck", cmethod)
            else:
                remote_gb, dbfile = self._construct_paths(
                    uri, "dblight", cmethod)
                remote_gb, md5file = self._construct_paths(
                    uri, "cklight", cmethod)

        elif self._repo_eapi == 2:
            remote_gb, dbfile = self._construct_paths(
                uri, "dbdumplight", cmethod)
            remote_gb, md5file = self._construct_paths(
                uri, "dbdumplightck", cmethod)

        else:
            raise AttributeError("EAPI must be = 1 or 2")

        try:
            return self.__verify_file_checksum(dbfile, md5file)
        except (OSError, IOError) as err:
            if err.errno != errno.ENOENT:
                raise
            return -1

    def __unpack_downloaded_database(self, uri, down_item, cmethod):

        rc = 0
        path = None
        sys_set_repos = self._settings['repositories']['available']
        repo_data = sys_set_repos[self._repository_id]

        garbage, myfile = self._construct_paths(uri, down_item, cmethod)

        if self._repo_eapi in (1, 2,):
            try:

                myfunc = getattr(entropy.tools, cmethod[1])
                path = myfunc(myfile)
                # rename path correctly
                if self._repo_eapi == 1:
                    new_path = os.path.join(os.path.dirname(path),
                        etpConst['etpdatabasefile'])
                    os.rename(path, new_path)
                    path = new_path

            except (OSError, EOFError):
                rc = 1

        else:
            mytxt = "invalid EAPI must be = 1 or 2"
            raise AttributeError(mytxt)

        if rc == 0:
            const_setup_file(path, etpConst['entropygid'], 0o644,
                uid = etpConst['uid'])

        return rc

    def _downloaded_database_unpack(self, uri, cmethod):
        """
        Unpack the downloaded database.
        """
        file_to_unpack = etpConst['etpdatabasedump']
        if self._repo_eapi == 1:
            file_to_unpack = etpConst['etpdatabasefile']
        elif self._repo_eapi == 2:
            file_to_unpack = etpConst['etpdatabasedumplight']

        mytxt = "%s %s %s" % (red(_("Unpacking database to")),
            darkgreen(file_to_unpack), red("..."),)
        self._entropy.output(
            mytxt,
            importance = 0,
            level = "info",
            header = "\t"
        )

        myitem = 'dblight'
        if self._repo_eapi == 2:
            myitem = 'dbdumplight'
        elif self._developer_repo:
            myitem = 'db'

        myrc = self.__unpack_downloaded_database(uri, myitem, cmethod)
        if myrc != 0:
            mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")),
                red(_("Skipping repository")),)
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = "\t"
            )
            return False, myitem
        return True, myitem

    def __update_repository_revision(self, revision):
        cur_rev = AvailablePackagesRepository.revision(self._repository_id)
        repo_data = self._settings['repositories']
        db_data = repo_data['available'][self._repository_id]
        db_data['dbrevision'] = "0"
        if cur_rev != -1:
            db_data['dbrevision'] = str(cur_rev)

        rev_file = os.path.join(db_data['dbpath'],
            etpConst['etpdatabaserevisionfile'])
        enc = etpConst['conf_encoding']
        with codecs.open(rev_file, "w", encoding=enc) as rev_f:
            # safe anyway
            rev_f.write("%s\n" % (revision,))
            rev_f.flush()

    def __validate_database(self):
        def tell_error(err):
            mytxt = "%s: %s" % (darkred(_("Repository is invalid")),
                repr(err),)
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "error",
                header = "\t"
            )
        try:
            dbconn = self._entropy.open_repository(self._repository_id)
        except RepositoryError as err:
            tell_error(err)
            return False
        try:
            dbconn.validate()
        except SystemDatabaseError as err:
            tell_error(err)
            return False
        return True

    def __database_indexing(self):

        # renice a bit, to avoid eating resources
        old_prio = const_set_nice_level(15)
        mytxt = red("%s ...") % (_("Indexing Repository metadata"),)
        self._entropy.output(
            mytxt,
            importance = 1,
            level = "info",
            header = "\t"
        )
        dbconn = self._entropy.open_repository(self._repository_id)
        dbconn.createAllIndexes()
        dbconn.commit(force = True)

        inst_repo = self._entropy.installed_repository()
        if inst_repo is not None:
            with inst_repo.exclusive():
                try: # client db can be absent
                    inst_repo.createAllIndexes()
                except (DatabaseError, OperationalError, IntegrityError,):
                    pass
        const_set_nice_level(old_prio)

    def _construct_paths(self, uri, item, cmethod, get_signature = False):
        """
        Build a remote URL and a local path for a supported resource item.
        """
        if item not in self._supported_download_items:
            raise AttributeError("Invalid item: %s" % (item,))

        items_needing_cmethod = (
            "db", "dbck", "dblight", "cklight", "dbdump", "dbdumpck",
            "dbdumplight", "dbdumplightck", "compck",
        )
        if (item in items_needing_cmethod) and (cmethod is None):
            mytxt = "For %s, cmethod can't be None" % (item,)
            raise AttributeError(mytxt)

        avail_data = self._settings['repositories']['available']
        repo_data = avail_data[self._repository_id]

        repo_dbpath = repo_data['dbpath']
        ec_hash = etpConst['etpdatabasehashfile']
        repo_lock_file = etpConst['etpdatabasedownloadlockfile']
        notice_board_filename = os.path.basename(repo_data['notice_board'])
        meta_file = etpConst['etpdatabasemetafilesfile']
        meta_file_gpg = etpConst['etpdatabasemetafilesfile'] + \
            etpConst['etpgpgextension']
        md5_ext = etpConst['packagesmd5fileext']
        ec_cm2 = None
        ec_cm3 = None
        ec_cm4 = None
        ec_cm5 = None
        ec_cm6 = None
        ec_cm7 = None
        ec_cm8 = None
        ec_cm9 = None
        if cmethod is not None:
            ec_cm2 = etpConst[cmethod[2]]
            ec_cm3 = etpConst[cmethod[3]]
            ec_cm4 = etpConst[cmethod[4]]
            ec_cm5 = etpConst[cmethod[5]]
            ec_cm6 = etpConst[cmethod[6]]
            ec_cm7 = etpConst[cmethod[7]]
            ec_cm8 = etpConst[cmethod[8]]
            ec_cm9 = etpConst[cmethod[9]]

        mymap = {
            'db': (
                "%s/%s" % (uri, ec_cm2,),
                "%s/%s" % (repo_dbpath, ec_cm2,),
            ),
            'dbck': (
                "%s/%s" % (uri, ec_cm9,),
                "%s/%s" % (repo_dbpath, ec_cm9,),
            ),
            'dblight': (
                "%s/%s" % (uri, ec_cm7,),
                "%s/%s" % (repo_dbpath, ec_cm7,),
            ),
            'dbdump': (
                "%s/%s" % (uri, ec_cm3,),
                "%s/%s" % (repo_dbpath, ec_cm3,),
            ),
            'dbdumplight': (
                "%s/%s" % (uri, ec_cm5,),
                "%s/%s" % (repo_dbpath, ec_cm5,),
            ),
            'ck': (
                "%s/%s" % (uri, ec_hash,),
                "%s/%s" % (repo_dbpath, ec_hash,),
            ),
            'cklight': (
                "%s/%s" % (uri, ec_cm8,),
                "%s/%s" % (repo_dbpath, ec_cm8,),
            ),
            'compck': (
                "%s/%s%s" % (uri, ec_cm2, md5_ext,),
                "%s/%s%s" % (repo_dbpath, ec_cm2, md5_ext,),
            ),
            'dbdumpck': (
                "%s/%s" % (uri, ec_cm4,),
                "%s/%s" % (repo_dbpath, ec_cm4,),
            ),
            'dbdumplightck': (
                "%s/%s" % (uri, ec_cm6,),
                "%s/%s" % (repo_dbpath, ec_cm6,),
            ),
            'lock': (
                "%s/%s" % (uri, repo_lock_file,),
                "%s/%s" % (repo_dbpath, repo_lock_file,),
            ),
            'notice_board': (
                repo_data['notice_board'],
                "%s/%s" % (repo_dbpath, notice_board_filename,),
            ),
            'meta_file': (
                "%s/%s" % (uri, meta_file,),
                "%s/%s" % (repo_dbpath, meta_file,),
            ),
            'meta_file_gpg': (
                "%s/%s" % (uri, meta_file_gpg,),
                "%s/%s" % (repo_dbpath, meta_file_gpg,),
            ),
        }

        url, path = mymap.get(item)
        if get_signature:
            url = self.__append_gpg_signature_to_path(url)
            path = self.__append_gpg_signature_to_path(path)

        return url, path

    def _download_item(self, uri, item, cmethod = None,
        disallow_redirect = True, get_signature = False):

        url, filepath = self._construct_paths(
            uri, item, cmethod, get_signature = get_signature)

        # See bug #3495, download the file to
        # a temporary location and then move it
        # if we are successful
        temp_filepath = filepath + ".edownload"

        # to avoid having permissions issues
        # it's better to remove the file before,
        # otherwise new permissions won't be written
        if os.path.isfile(temp_filepath):
            os.remove(temp_filepath)
        filepath_dir = os.path.dirname(temp_filepath)

        if not os.path.isdir(filepath_dir) and not \
            os.path.lexists(filepath_dir):

            os.makedirs(filepath_dir, 0o755)
            const_setup_perms(filepath_dir, etpConst['entropygid'],
                f_perms = 0o644)

        try:

            fetcher = self._entropy._url_fetcher(
                url,
                temp_filepath,
                resume = False,
                disallow_redirect = disallow_redirect
            )

            rc = fetcher.download()
            if rc in self.FETCH_ERRORS:
                return False
            try:
                os.rename(temp_filepath, filepath)
            except (OSError, IOError) as err:
                if err.errno != errno.ENOENT:
                    raise
                return False # not downloaded?

            const_setup_file(filepath, etpConst['entropygid'], 0o644,
                uid = etpConst['uid'])
            return True

        finally:
            # cleanup temp file
            try:
                os.remove(temp_filepath)
            except (OSError, IOError) as err:
                if err.errno != errno.ENOENT:
                    raise

    def _is_repository_unlocked(self, uri):
        """
        Returns whether the repository is remotely locked or not.

        @return: repository being remotely locked
        @rtype:bool
        """
        rc = self._download_item(uri, "lock", disallow_redirect = True)
        if rc: # cannot download database
            return False
        return True

    def _standard_items_download(self, uri):
        """
        Download a set of "standard" files from the repository mirror.
        """
        repos_data = self._settings['repositories']
        repo_data = repos_data['available'][self._repository_id]
        notice_board = os.path.basename(repo_data['local_notice_board'])
        db_meta_file = etpConst['etpdatabasemetafilesfile']
        db_meta_file_gpg = etpConst['etpdatabasemetafilesfile'] + \
            etpConst['etpgpgextension']

        objects_to_unpack = ("meta_file",)

        download_items = [
            (
                "meta_file",
                db_meta_file,
                False,
                "%s %s %s" % (
                    red(_("Downloading repository metafile")),
                    darkgreen(db_meta_file),
                    red("..."),
                )
            ),
            (
                "meta_file_gpg",
                db_meta_file_gpg,
                True,
                "%s %s %s" % (
                    red(_("Downloading GPG signature of repository metafile")),
                    darkgreen(db_meta_file_gpg),
                    red("..."),
                )
            ),
            (
                "notice_board",
                notice_board,
                True,
                "%s %s %s" % (
                    red(_("Downloading Notice Board")),
                    darkgreen(notice_board),
                    red("..."),
                )
            ),
        ]

        def my_show_info(txt):
            self._entropy.output(
                txt,
                importance = 0,
                level = "info",
                header = "\t",
                back = True
            )

        def my_show_down_status(message, mytype):
            self._entropy.output(
                message,
                importance = 0,
                level = mytype,
                header = "\t"
            )

        def my_show_file_unpack(fp):
            self._entropy.output(
                "%s: %s" % (darkgreen(_("unpacked meta file")), brown(fp),),
                header = blue("\t  << ")
            )

        def my_show_file_rm(fp):
            self._entropy.output(
                "%s: %s" % (darkgreen(_("removed meta file")), purple(fp),),
                header = blue("\t  << ")
            )

        downloaded_files = []

        for item, myfile, ignorable, mytxt in download_items:

            my_show_info(mytxt)
            mystatus = self._download_item(uri, item, disallow_redirect = True)
            mytype = 'info'
            myurl, mypath = self._construct_paths(uri, item, None)

            # download failed, is it critical?
            if not mystatus:
                if ignorable:
                    message = "%s: %s." % (blue(myfile),
                        red(_("not available, it's ok")))
                else:
                    mytype = 'warning'
                    message = "%s: %s." % (blue(myfile),
                        darkred(_("not available, not very ok!")))
                my_show_down_status(message, mytype)

                # remove garbage
                if os.path.isfile(mypath):
                    try:
                        os.remove(mypath)
                    except OSError:
                        continue

                continue

            message = "%s: %s." % (blue(myfile),
                darkgreen(_("available, w00t!")))
            my_show_down_status(message, mytype)
            downloaded_files.append(mypath)

            if item not in objects_to_unpack:
                continue
            if not const_file_readable(mypath):
                continue

            tmpdir = const_mkdtemp(prefix="_standard_items_download")
            repo_dir = repo_data['dbpath']
            enc = etpConst['conf_encoding']
            try:
                done = entropy.tools.universal_uncompress(mypath, tmpdir,
                    catch_empty = True)
                if not done:
                    mytype = 'warning'
                    message = "%s: %s." % (blue(myfile),
                        darkred(_("cannot be unpacked, not very ok!")))
                    my_show_down_status(message, mytype)
                    continue
                myfiles_to_move = set(os.listdir(tmpdir))

                # exclude files not available by default
                files_not_found_file = etpConst['etpdatabasemetafilesnotfound']
                if files_not_found_file in myfiles_to_move:
                    myfiles_to_move.remove(files_not_found_file)
                    fnf_path = os.path.join(tmpdir, files_not_found_file)

                    f_nf = []
                    try:
                        with codecs.open(fnf_path, "r", encoding=enc) as f:
                            f_nf += [x.strip() for x in f.readlines()]
                    except (OSError, IOError) as err:
                        if err.errno != errno.ENOENT:
                            raise
                    else:
                        for myfile in f_nf:
                            myfile = os.path.basename(myfile) # avoid lamerz
                            myfpath = os.path.join(repo_dir, myfile)
                            try:
                                os.remove(myfpath)
                                my_show_file_rm(myfile)
                            except (OSError, IOError) as err:
                                if err.errno != errno.ENOENT:
                                    raise

                for myfile in sorted(myfiles_to_move):
                    from_mypath = os.path.join(tmpdir, myfile)
                    to_mypath = os.path.join(repo_dir, myfile)
                    try:
                        os.rename(from_mypath, to_mypath)
                        my_show_file_unpack(myfile)
                    except OSError:
                        # try non atomic way
                        try:
                            shutil.copy2(from_mypath, to_mypath)
                            my_show_file_unpack(myfile)
                            os.remove(from_mypath)
                        except (shutil.Error, IOError, OSError,):
                            continue
                        continue

                    const_setup_file(to_mypath, etpConst['entropygid'], 0o644,
                        uid = etpConst['uid'])

            finally:
                shutil.rmtree(tmpdir, True)

        repo_r = AvailablePackagesRepository.revision(self._repository_id)
        mytxt = "%s: %s" % (
            red(_("Repository revision")),
            bold(str(repo_r)),
        )
        self._entropy.output(
            mytxt,
            importance = 1,
            level = "info",
            header = "\t"
        )

        return downloaded_files

    def _check_downloaded_database(self, uri, cmethod):

        dbitem = "dblight"
        if self._repo_eapi == 2:
            dbitem = "dbdumplight"
        elif self._developer_repo:
            dbitem = "db"
        garbage, dbfilename = self._construct_paths(uri, dbitem, cmethod)

        # verify checksum
        mytxt = "%s %s %s" % (
            red(_("Checking downloaded repository")),
            darkgreen(os.path.basename(dbfilename)),
            red("..."),
        )
        self._entropy.output(
            mytxt,
            importance = 0,
            back = True,
            level = "info",
            header = "\t"
        )
        db_status = self.__verify_database_checksum(uri, cmethod)
        if db_status == -1:
            mytxt = "%s. %s !" % (
                red(_("Cannot open digest")),
                red(_("Cannot verify repository integrity")),
            )
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = "\t"
            )
        elif db_status:
            mytxt = "%s: %s" % (
                red(_("Downloaded repository status")),
                bold(_("OK")),
            )
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "info",
                header = "\t"
            )
        else:
            mytxt = "%s: %s" % (
                red(_("Downloaded repository status")),
                darkred(_("ERROR")),
            )
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "error",
                header = "\t"
            )
            mytxt = "%s. %s" % (
                red(_("An error occurred while checking repository integrity")),
                red(_("Giving up")),
            )
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "error",
                header = "\t"
            )
            return 1
        return 0

    def _install_gpg_key_if_available(self):

        my_repos = self._settings['repositories']
        avail_data = my_repos['available']
        repo_data = avail_data[self._repository_id]
        gpg_path = repo_data['gpg_pubkey']

        if not const_file_readable(gpg_path):
            return False # gpg key not available

        def do_warn_user(fingerprint):
            mytxt = purple(_("Make sure to verify the imported key and "
                             "set an appropriate trust level"))
            self._entropy.output(
                mytxt + ":",
                level = "warning",
                header = "\t"
            )
            mytxt = brown("gpg --homedir '%s' --edit-key '%s'" % (
                etpConst['etpclientgpgdir'], fingerprint,)
            )
            self._entropy.output(
                "$ " + mytxt,
                level = "warning",
                header = "\t"
            )

        try:
            repo_sec = self._entropy.RepositorySecurity()
        except RepositorySecurity.GPGError:
            mytxt = "%s," % (
                purple(_("This repository suports GPG-signed packages")),
            )
            self._entropy.output(
                mytxt,
                level = "warning",
                header = "\t"
            )
            mytxt = purple(_("you may want to install GnuPG to take "
                             "advantage of this feature"))
            self._entropy.output(
                mytxt,
                level = "warning",
                header = "\t"
            )
            return False # GPG not available

        pk_expired = False
        try:
            pk_avail = repo_sec.is_pubkey_available(self._repository_id)
        except repo_sec.KeyExpired:
            pk_avail = False
            pk_expired = True

        if pk_avail:

            tmp_dir = const_mkdtemp(prefix="_install_gpg_key")
            repo_tmp_sec = self._entropy.RepositorySecurity(
                keystore_dir = tmp_dir)
            # try to install and get fingerprint
            try:
                downloaded_key_fp = repo_tmp_sec.install_key(
                    self._repository_id, gpg_path)
            except RepositorySecurity.GPGError:
                downloaded_key_fp = None

            fingerprint = repo_sec.get_key_metadata(
                self._repository_id)['fingerprint']
            shutil.rmtree(tmp_dir, True)

            if downloaded_key_fp != fingerprint and \
                (downloaded_key_fp is not None):
                mytxt = "%s: %s !!!" % (
                    purple(_("GPG key changed for")),
                    bold(self._repository_id),
                )
                self._entropy.output(
                    mytxt,
                    level = "warning",
                    header = "\t"
                )
                mytxt = "[%s => %s]" % (
                    darkgreen(fingerprint),
                    purple(downloaded_key_fp),
                )
                self._entropy.output(
                    mytxt,
                    level = "warning",
                    header = "\t"
                )
                do_warn_user(downloaded_key_fp)
            else:
                mytxt = "%s: %s" % (
                    purple(_("GPG key already installed for")),
                    bold(self._repository_id),
                )
                self._entropy.output(
                    mytxt,
                    level = "info",
                    header = "\t"
                )
                do_warn_user(fingerprint)
                return True # already installed

        elif pk_expired:
            mytxt = "%s: %s" % (
                purple(_("GPG key EXPIRED for repository")),
                bold(self._repository_id),
            )
            self._entropy.output(
                mytxt,
                level = "warning",
                header = "\t"
            )


        # actually install
        mytxt = "%s: %s" % (
            purple(_("Installing GPG key for repository")),
            brown(self._repository_id),
        )
        self._entropy.output(
            mytxt,
            level = "info",
            header = "\t",
            back = True
        )

        try_ignore = False
        while True:
            try:
                fingerprint = repo_sec.install_key(self._repository_id,
                    gpg_path)
            except RepositorySecurity.NothingImported as err:
                if try_ignore:
                    mytxt = "%s: %s" % (
                        darkred(_("Error during GPG key installation")),
                        err,
                    )
                    self._entropy.output(
                        mytxt,
                        level = "error",
                        header = "\t"
                    )
                    return False
                self._entropy.output(
                    purple(_("GPG key seems already installed but "
                             "not properly recorded, resetting")),
                    level = "warning",
                    header = "\t"
                )
                target_fingerprint = repo_sec.get_key_fingerprint(gpg_path)
                if target_fingerprint is not None:
                    # kill it, this is usually caused by shadow repos
                    # like sabayon-weekly
                    dead_repository_ids = set()
                    for _repository_id, key_meta in repo_sec.get_keys().items():
                        if key_meta['fingerprint'] == target_fingerprint:
                            dead_repository_ids.add(_repository_id)
                    for _repository_id in dead_repository_ids:
                        try:
                            repo_sec.delete_pubkey(_repository_id)
                        except KeyError:
                            # wtf, fault tolerance
                            pass
                try_ignore = True
                continue
            except RepositorySecurity.GPGError as err:
                mytxt = "%s: %s" % (
                    darkred(_("Error during GPG key installation")),
                    err,
                )
                self._entropy.output(
                    mytxt,
                    level = "error",
                    header = "\t"
                )
                return False
            break

        mytxt = "%s: %s" % (
            purple(_("Successfully installed GPG key for repository")),
            brown(self._repository_id),
        )
        self._entropy.output(
            mytxt,
            level = "info",
            header = "\t"
        )
        mytxt = "%s: %s" % (
            darkgreen(_("Fingerprint")),
            bold(fingerprint),
        )
        self._entropy.output(
            mytxt,
            level = "info",
            header = "\t"
        )
        do_warn_user(fingerprint)
        return True

    def _gpg_verify_downloaded_files(self, downloaded_files):

        try:
            repo_sec = self._entropy.RepositorySecurity()
        except RepositorySecurity.GPGServiceNotAvailable:
            # wtf! it was available a while ago!
            return 0 # GPG not available

        gpg_sign_ext = etpConst['etpgpgextension']
        sign_files = [x for x in downloaded_files if \
            x.endswith(gpg_sign_ext)]
        sign_files = [x for x in sign_files if const_file_readable(x)]

        to_be_verified = []

        for sign_path in sign_files:
            target_path = sign_path[:-len(gpg_sign_ext)]
            if const_file_readable(target_path):
                to_be_verified.append((target_path, sign_path,))

        gpg_rc = 0

        for target_path, sign_path in to_be_verified:

            file_name = os.path.basename(target_path)

            mytxt = "%s: %s ..." % (
                darkgreen(_("Verifying GPG signature of")),
                brown(file_name),
            )
            self._entropy.output(
                mytxt,
                level = "info",
                header = blue("\t@@ "),
                back = True
            )

            is_valid, err_msg = repo_sec.verify_file(self._repository_id,
                target_path, sign_path)
            if is_valid:
                mytxt = "%s: %s" % (
                    darkgreen(_("Verified GPG signature of")),
                    brown(file_name),
                )
                self._entropy.output(
                    mytxt,
                    level = "info",
                    header = blue("\t@@ ")
                )
            else:
                mytxt = "%s: %s" % (
                    darkred(_("Error during GPG verification of")),
                    file_name,
                )
                self._entropy.output(
                    mytxt,
                    level = "error",
                    header = "\t%s " % (bold("!!!"),)
                )
                mytxt = "%s: %s" % (
                    purple(_("It could mean a potential security risk")),
                    err_msg,
                )
                self._entropy.output(
                    mytxt,
                    level = "error",
                    header = "\t%s " % (bold("!!!"),)
                )
                gpg_rc = 1

        return gpg_rc

    def _webservice_database_sync(self):
        """
        Update the local repository database through the webservice
        using a differential sync.
        """
        repo_db = None
        try:
            repo_db = self.__get_webserv_local_database()
            if repo_db is None:
                raise AttributeError()
            return self.__handle_webserv_database_sync(repo_db)
        except (DatabaseError, IntegrityError, OperationalError,
            AttributeError,):
            return False
        finally:
            if repo_db is not None:
                repo_db.commit()
                repo_db.close()

        return False

    def __handle_webserv_database_sync(self, mydbconn):

        try:
            webserv = self._webservice
        except WebService.UnsupportedService as err:
            const_debug_write(__name__,
                "__handle_webserv_database_sync: error: %s" % (err,))
            return False

        try:
            mypackage_ids = mydbconn.listAllPackageIds()
        except (DatabaseError, IntegrityError, OperationalError,):
            return False

        added_ids, removed_ids = self.__get_webserv_database_differences(
            webserv, mypackage_ids)
        if (None in (added_ids, removed_ids)) or \
            (not added_ids and not removed_ids and self.__force):
            # nothing to sync, it seems, if force is True, fallback to EAPI2
            return False

        threshold = 100
        # is it worth it?
        if len(added_ids) > threshold:
            mytxt = "%s: %s (%s: %s/%s)" % (
                blue(_("Web Service")),
                darkred(_("skipping differential sync")),
                brown(_("threshold")),
                blue(str(len(added_ids))),
                darkred(str(threshold)),
            )
            self._entropy.output(
                mytxt,
                importance = 0,
                level = "info",
                header = blue("  # "),
            )
            return False

        count = 0
        chunk_size = RepositoryWebService.MAXIMUM_PACKAGE_REQUEST_SIZE
        added_segments = []
        mytmp = []

        for package_id in added_ids:
            count += 1
            mytmp.append(package_id)
            if count % chunk_size == 0:
                added_segments.append(mytmp[:])
                del mytmp[:]
        if mytmp:
            added_segments.append(mytmp[:])

        def _do_fetch(fetch_sts_map, segment, count, maxcount):
            try:
                try:
                    pkg_meta = webserv.get_packages_metadata(segment)
                except WebService.WebServiceException as err:
                    const_debug_write(__name__,
                        "__handle_webserv_database_sync: error: %s" % (err,))
                    mytxt = "%s: %s" % (
                        blue(_("Web Service communication error")),
                        err,
                    )
                    self._entropy.output(
                        mytxt, importance = 1, level = "info",
                        header = "\t", count = (count, maxcount,)
                    )
                    fetch_sts_map['error'] = True
                    return
                except KeyboardInterrupt:
                    const_debug_write(__name__,
                        "__handle_webserv_database_sync: keyboard interrupt")
                    fetch_sts_map['error'] = True
                    return

                if not pkg_meta:
                    const_debug_write(__name__,
                        "__handle_webserv_database_sync: empty data: %s" % (
                            pkg_meta,))
                    self._entropy.output(
                        _("Web Service data error"), importance = 1,
                        level = "info", header = "\t",
                        count = (count, maxcount,)
                    )
                    fetch_sts_map['error'] = True
                    return

                try:
                    for package_id, pkg_data in pkg_meta.items():
                        dumpobj(
                            "%s%s" % (self.WEBSERV_CACHE_ID, package_id,),
                            pkg_data,
                            ignore_exceptions = False
                        )
                except (IOError, EOFError, OSError,) as e:
                    mytxt = "%s: %s: %s." % (
                        blue(_("Local status")),
                        darkred("Error storing data"),
                        e,
                    )
                    self._entropy.output(
                        mytxt, importance = 1, level = "info",
                        header = "\t", count = (count, maxcount,)
                    )
                    fetch_sts_map['error'] = True
                    return

                return
            finally:
                fetch_sts_map['sem'].release()

        # do not exagerate or you're going to need a way to block
        # further requests as long as some threads are still running
        # to avoid timeout errors
        max_threads = 4
        fetch_sts_map = {
            'sem': threading.Semaphore(max_threads),
            'error': False,
        }

        # fetch and store
        count = 0
        maxcount = len(added_segments)
        product = self._settings['repositories']['product']
        segment = None
        threads = []
        for segment in added_segments:
            count += 1
            mytxt = "%s %s" % (blue(_("Fetching segments")), "...",)
            self._entropy.output(
                mytxt, importance = 0, level = "info",
                header = "\t", back = True, count = (count, maxcount,)
            )
            fetch_sts_map['sem'].acquire()
            if len(threads) >= max_threads:
                const_debug_write(__name__,
                    purple("joining all the parallel threads"))
                # give them the chance to complete
                # since long delays on socket could cause timeouts
                for th in threads:
                    th.join()
                const_debug_write(__name__, purple("parallel threads joined"))
                del threads[:]
            if fetch_sts_map['error']:
                return None
            th = ParallelTask(_do_fetch, fetch_sts_map, segment, count,
                maxcount)
            th.daemon = True
            th.start()
            threads.append(th)

        for th in threads:
            th.join()

        if fetch_sts_map['error']:
            return None

        del added_segments

        # get repository metadata
        repo_metadata = self.__get_webserv_repository_metadata()
        # this gives us the "checksum" data too
        if not repo_metadata:
            mytxt = "%s: %s" % (
                blue(_("Web Service status")),
                darkred(_("cannot fetch repository metadata")),
            )
            self._entropy.output(
                mytxt,
                importance = 0,
                level = "info",
                header = blue("  # "),
            )
            return None

        # update treeupdates
        try:
            mydbconn.setRepositoryUpdatesDigest(self._repository_id,
                repo_metadata['treeupdates_digest'])
            mydbconn.bumpTreeUpdatesActions(
                repo_metadata['treeupdates_actions'])
        except (Error,):
            mytxt = "%s: %s" % (
                blue(_("Web Service status")),
                darkred(_("cannot update treeupdates data")),
            )
            self._entropy.output(
                mytxt,
                importance = 0,
                level = "info",
                header = blue("  # "),
            )
            return None

        # update package sets
        try:
            mydbconn.clearPackageSets()
            mydbconn.insertPackageSets(repo_metadata['sets'])
        except (Error,):
            mytxt = "%s: %s" % (
                blue(_("Web Service status")),
                darkred(_("cannot update package sets data")),
            )
            self._entropy.output(
                mytxt,
                importance = 0,
                level = "info",
                header = blue("  # "),
            )
            return None

        # now that we have all stored, add
        for package_id in added_ids:
            mydata = self._cacher.pop("%s%s" % (self.WEBSERV_CACHE_ID,
                package_id,))
            if mydata is None:
                mytxt = "%s: %s" % (
                    blue(_("Fetch error on segment while adding")),
                    darkred(str(segment)),
                )
                self._entropy.output(
                    mytxt, importance = 1, level = "warning",
                    header = "  "
                )
                return False

            mytxt = "%s %s" % (
                darkgreen("++"),
                teal(mydata['atom']),
            )
            self._entropy.output(
                mytxt, importance = 0, level = "info",
                header = "  ")
            try:
                mydbconn.addPackage(
                    mydata, revision = mydata['revision'],
                    package_id = package_id,
                    formatted_content = True
                )
            except (Error,) as err:
                if const_debug_enabled():
                    entropy.tools.print_traceback()
                self._entropy.output("%s: %s" % (
                    blue(_("repository error while adding packages")),
                    err,),
                    importance = 1, level = "warning",
                    header = "  "
                )
                return False

        # now remove
        # preload atoms names to improve speed during removePackage
        atoms_map = dict((x, mydbconn.retrieveAtom(x),) for x in removed_ids)
        for package_id in removed_ids:
            myatom = atoms_map.get(package_id)

            mytxt = "%s %s" % (
                darkred("--"),
                purple(str(myatom)),)
            self._entropy.output(
                mytxt, importance = 0, level = "info",
                header = "  ")
            try:
                mydbconn.removePackage(package_id)
            except (Error,):
                self._entropy.output(
                    blue(_("repository error while removing packages")),
                    importance = 1, level = "warning",
                    header = "  "
                )
                return False

        mydbconn.commit()
        mydbconn.clearCache()
        # now verify if both checksums match
        result = False
        mychecksum = mydbconn.checksum(do_order = True,
            strict = False, include_signatures = True)
        if repo_metadata['checksum'] == mychecksum:
            result = True
        else:
            self._entropy.output(
                blue(_("Repository checksum doesn't match remote.")),
                importance = 0, level = "info", header = "\t",
            )
            mytxt = "%s: %s" % (_('local'), mychecksum,)
            self._entropy.output(
                mytxt, importance = 0,
                level = "info", header = "\t",
            )
            mytxt = "%s: %s" % (_('remote'), repo_metadata['checksum'],)
            self._entropy.output(
                mytxt, importance = 0,
                level = "info", header = "\t",
            )

        return result

    def remote_revision(self):
        rev = self._remote_webservice_revision()
        if rev is not None:
            return rev

        avail_data = self._settings['repositories']['available']
        repo_data = avail_data[self._repository_id]
        # default to the first entry, which is expected
        # to be always available.
        try:
            uri = repo_data['databases'][0]['uri']
        except IndexError:
            rev = -1
            return rev

        rev = self._remote_revision(uri)
        return rev

    def _remote_webservice_revision(self):
        """
        Return the remote repository revision using the webservice.
        This method returns None if the request failed or the
        webservice is not available.
        """
        if self._repo_eapi < 3:
            return

        # ask WebService then
        revision = self.__get_webserv_repository_revision()
        if revision is not None:
            try:
                revision = int(revision)
            except ValueError:
                revision = None

        if revision is not None:
            return revision

        # otherwise, fallback to previous EAPI
        self._repo_eapi -= 1

    def _remote_revision(self, uri):
        """
        Return the remote repository revision by downloading
        the revision file from the given uri.
        """
        sep = const_convert_to_unicode("/")
        url = uri + sep + etpConst['etpdatabaserevisionfile']

        tmp_fd, tmp_path = None, None
        rev = "-1"
        try:
            tmp_fd, tmp_path = const_mkstemp(
                prefix = "AvailableEntropyRepository.remote_revision")
            fetcher = self._entropy._url_fetcher(
                url, tmp_path, resume = False)
            fetch_rc = fetcher.download()
            if fetch_rc not in self.FETCH_ERRORS:
                with codecs.open(tmp_path, "r") as tmp_f:
                    rev = tmp_f.readline().strip()
        except (IOError, OSError):
            # ignore any errors, especially read ones
            pass
        finally:
            if tmp_fd is not None:
                try:
                    os.close(tmp_fd)
                except OSError:
                    pass
            if tmp_path is not None:
                try:
                    os.remove(tmp_path)
                except OSError:
                    pass

        # try to convert rev into integer now
        try:
            rev = int(rev)
        except ValueError:
            # corrupted data
            rev = -1

        return rev

    def update(self):

        # disallow unprivileged update
        if not entropy.tools.is_root():
            raise PermissionDenied(
                "cannot update repository as unprivileged user")

        self._show_repository_information()

        selected = self._select_database_mirror()
        if selected is None:
            return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
        revision, uri, cformat = selected

        updatable = self._is_repository_updatable(revision)
        if not self.__force:
            if not updatable:
                mytxt = "%s: %s." % (bold(_("Attention")),
                    red(_("repository is already up to date")),)
                self._entropy.output(
                    mytxt,
                    importance = 1,
                    level = "info",
                    header = "\t"
                )
                return EntropyRepositoryBase.REPOSITORY_ALREADY_UPTODATE

        locked = not self._is_repository_unlocked(uri)
        if locked:
            mytxt = "%s: %s. %s." % (
                bold(_("Attention")),
                red(_("Repository is being updated")),
                red(_("Try again in a few minutes")),
            )
            self._entropy.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = "\t"
            )
            return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE

        # clear database interface cache belonging to this repository
        self._ensure_repository_path()

        # dealing with EAPI
        # setting some vars
        db_checksum_down_status = False
        do_db_update_transfer = False
        rc = 0

        my_repos = self._settings['repositories']
        avail_data = my_repos['available']
        repo_data = avail_data[self._repository_id]

        # some variables
        dumpfile = os.path.join(repo_data['dbpath'],
            etpConst['etpdatabasedumplight'])
        dbfile = os.path.join(repo_data['dbpath'],
            etpConst['etpdatabasefile'])
        dbfile_old = dbfile+".sync"
        cmethod = etpConst['etpdatabasecompressclasses'].get(
            cformat)

        while True:

            downloaded_db_item = None
            sig_down_status = False
            db_checksum_down_status = False
            if self._repo_eapi < 3:

                down_status, sig_down_status, downloaded_db_item = \
                    self.__database_download(uri, cmethod)
                if not down_status:
                    return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
                db_checksum_down_status = \
                    self.__database_checksum_download(uri, cmethod)
                break

            elif self._repo_eapi == 3 and not const_file_writable(dbfile):
                do_db_update_transfer = None
                self._repo_eapi -= 1
                continue

            elif self._repo_eapi == 3:

                status = False
                try:
                    status = self._webservice_database_sync()
                except:
                    # avoid broken entries, deal with every exception
                    entropy.tools.print_traceback()
                    self.__remove_repository_files()
                    raise

                if not status:
                    # set to none and completely skip database alignment
                    do_db_update_transfer = None
                    self._repo_eapi -= 1
                    continue

                break

        downloaded_files = self._standard_items_download(uri)
        # also add db file to downloaded item
        # and md5 check repository
        if downloaded_db_item is not None:

            durl, dpath = self._construct_paths(
                uri, downloaded_db_item, cmethod)
            downloaded_files.append(dpath)
            if sig_down_status:
                d_sig_path = self.__append_gpg_signature_to_path(dpath)
                downloaded_files.append(d_sig_path)

            # 1. we're always in EAPI1 or 2 here
            # 2. new policy, always deny repository if
            #    its database checksum cannot be fetched
            if not db_checksum_down_status:
                # delete all
                self.__remove_repository_files()
                return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE

            rc = self._check_downloaded_database(uri, cmethod)
            if rc != 0:
                # delete all
                self.__remove_repository_files()
                return EntropyRepositoryBase.REPOSITORY_CHECKSUM_ERROR

        # GPG pubkey install hook
        if self._gpg_feature:
            gpg_available = self._install_gpg_key_if_available()
            if gpg_available:
                gpg_rc = self._gpg_verify_downloaded_files(downloaded_files)

        # Now we can unpack
        files_to_remove = []
        if self._repo_eapi in (1, 2,):

            # if do_db_update_transfer == False and not None
            if (do_db_update_transfer is not None) and not \
                do_db_update_transfer:

                try:
                    os.rename(dbfile, dbfile_old)
                    do_db_update_transfer = True
                except OSError as err:
                    const_debug_write(
                        __name__, "rename failed: %s" % (err,))
                    do_db_update_transfer = False

            unpack_status, unpacked_item = \
                self._downloaded_database_unpack(uri, cmethod)

            if not unpack_status:
                # delete all
                self.__remove_repository_files()
                return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR

            unpack_url, unpack_path = self._construct_paths(
                uri, unpacked_item, cmethod)
            files_to_remove.append(unpack_path)

            # re-validate
            if not os.path.isfile(dbfile):
                do_db_update_transfer = False

            elif os.path.isfile(dbfile) and not do_db_update_transfer and \
                (self._repo_eapi != 1):
                os.remove(dbfile)

            if self._repo_eapi == 2:
                rc = self.__eapi2_inject_downloaded_dump(dumpfile,
                    dbfile, cmethod)

            if do_db_update_transfer:
                self.__eapi1_eapi2_databases_alignment(dbfile, dbfile_old)

            if self._repo_eapi == 2:
                # remove the dump
                files_to_remove.append(dumpfile)

        if rc != 0:
            # delete all
            self.__remove_repository_files()
            files_to_remove.append(dbfile_old)
            for path in files_to_remove:
                try:
                    os.remove(path)
                except OSError:
                    continue
            return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR

        # make sure that all the repository files are stored with proper
        # permissions to avoid possible XSS and trust boundary problems.
        downloaded_files.append(dbfile)
        for downloaded_file in sorted(set(downloaded_files)):
            try:
                const_setup_file(downloaded_file,
                                 etpConst['entropygid'], 0o644,
                                 uid = etpConst['uid'])
            except (OSError, IOError) as err:
                if err.errno != errno.ENOENT:
                    raise

        # remove garbage left around
        for path in files_to_remove:
            try:
                os.remove(path)
            except OSError:
                continue

        valid = self.__validate_database()
        if not valid:
            # repository failed validation
            return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR

        self.__update_repository_revision(revision)
        if self._entropy._indexing:
            self.__database_indexing()

        try:
            spm_class = self._entropy.Spm_class()
            spm_class.entropy_client_post_repository_update_hook(
                self._entropy, self._repository_id)
        except Exception as err:
            entropy.tools.print_traceback()
            mytxt = "%s: %s" % (
                blue(_("Configuration files update error, "
                       "not critical, continuing")),
                err,
            )
            self._entropy.output(mytxt, importance = 0,
                level = "info", header = blue("  # "),)

        # remove garbage
        try:
            os.remove(dbfile_old)
        except OSError:
            pass

        return EntropyRepositoryBase.REPOSITORY_UPDATED_OK


class MaskableRepository(EntropyRepositoryBase):
    """
    Objects inheriting from this class support package masking.
    A masked package is a package that is not visible to user and thus not
    selectable in dependency calculation and also not directly installable.
    The only repositories that need to support the feature are those containing
    installable packages, like AvailablePackagesRepository.
    """

    _real_client_settings = None
    _real_client_settings_lock = threading.Lock()

    def __init__(self, *args, **kwargs):
        super(MaskableRepository, self).__init__(*args, **kwargs)

    @property
    def _client_settings(self):
        """
        Load the Entropy Client settings object.
        """
        if self._real_client_settings is None:
            with self._real_client_settings_lock:
                if self._real_client_settings is None:
                    from entropy.client.interfaces import Client

                    self._real_client_settings = Client().ClientSettings()

        return self._real_client_settings

    @property
    def _settings_client_plugin(self):
        """
        Get the ClientSystemSettingsPlugin instance from Entropy Client.
        """
        from entropy.client.interfaces import Client
        return Client()._settings_client_plugin

    def _mask_filter_fetch_cache(self, package_id):
        if self._caching:
            return loadobj(
                "MaskableRepositoryFilter/%s_%s/%s" % (
                    self.name,
                    self.atomMatchCacheKey(),
                    package_id,
                    )
                )

    def _mask_filter_store_cache(self, package_id, value):
        if self._caching:
            dumpobj(
                "MaskableRepositoryFilter/%s_%s/%s" % (
                    self.name,
                    self.atomMatchCacheKey(),
                    package_id,
                    ),
                value)

    def _maskFilter_live(self, package_id):

        ref = self._settings['pkg_masking_reference']
        if (package_id, self.name) in \
            self._settings['live_packagemasking']['mask_matches']:

            # do not cache this
            return -1, ref['user_live_mask']

        elif (package_id, self.name) in \
            self._settings['live_packagemasking']['unmask_matches']:

            return package_id, ref['user_live_unmask']

    def _maskFilter_user_package_mask(self, package_id, live):

        with self._settings['mask']:
            # thread-safe in here
            cache_obj = self._settings['mask'].get()
            if cache_obj is None:
                cache_obj = {}
                self._settings['mask'].set(cache_obj)
        user_package_mask_ids = cache_obj.get(self.name)

        if user_package_mask_ids is None:
            user_package_mask_ids = set()

            for atom in self._settings['mask']:
                atom, repository_ids = entropy.dep.dep_get_match_in_repos(atom)
                if repository_ids is not None:
                    if self.name not in repository_ids:
                        # then the mask doesn't involve us
                        continue
                # check if @repository is specified
                matches, r = self.atomMatch(atom, multiMatch = True,
                    maskFilter = False)
                if r != 0:
                    continue
                user_package_mask_ids |= set(matches)

            cache_obj[self.name] = user_package_mask_ids

        if package_id in user_package_mask_ids:
            # sorry, masked
            ref = self._settings['pkg_masking_reference']
            myr = ref['user_package_mask']

            try:
                clset = self._client_settings
                validator_cache = clset['masking_validation']['cache']
                validator_cache[(package_id, self.name, live)] = -1, myr
            except KeyError: # system settings client plugin not found
                pass

            return -1, myr

    def _maskFilter_user_package_unmask(self, package_id, live):

        with self._settings['unmask']:
            # thread-safe in here
            cache_obj = self._settings['unmask'].get()
            if cache_obj is None:
                cache_obj = {}
                self._settings['unmask'].set(cache_obj)
        user_package_unmask_ids = cache_obj.get(self.name)

        if user_package_unmask_ids is None:

            user_package_unmask_ids = set()
            for atom in self._settings['unmask']:
                atom, repository_ids = entropy.dep.dep_get_match_in_repos(atom)
                if repository_ids is not None:
                    if self.name not in repository_ids:
                        # then the mask doesn't involve us
                        continue
                matches, r = self.atomMatch(atom, multiMatch = True,
                    maskFilter = False)
                if r != 0:
                    continue
                user_package_unmask_ids |= set(matches)

            cache_obj[self.name] = user_package_unmask_ids

        if package_id in user_package_unmask_ids:

            ref = self._settings['pkg_masking_reference']
            myr = ref['user_package_unmask']
            try:
                clset = self._client_settings
                validator_cache = clset['masking_validation']['cache']
                validator_cache[(package_id, self.name, live)] = \
                    package_id, myr
            except KeyError: # system settings client plugin not found
                pass

            return package_id, myr

    def _maskFilter_packages_db_mask(self, package_id, live):

        # check if repository packages.db.mask needs it masked
        repos_mask = {}
        clset = self._client_settings
        if clset:
            repos_mask = clset['repositories']['mask']

        repomask = repos_mask.get(self.name)
        if isinstance(repomask, (list, set, frozenset)):

            # first, seek into generic masking, all branches
            # (below) avoid issues with repository names
            mask_repo_id = "%s_ids@@:of:%s" % (self.name, self.name,)
            repomask_ids = repos_mask.get(mask_repo_id)

            if not isinstance(repomask_ids, set):
                repomask_ids = set()
                for atom in repomask:
                    matches, r = self.atomMatch(atom, multiMatch = True,
                        maskFilter = False)
                    if r != 0:
                        continue
                    repomask_ids |= set(matches)
                repos_mask[mask_repo_id] = repomask_ids

            if package_id in repomask_ids:

                ref = self._settings['pkg_masking_reference']
                myr = ref['repository_packages_db_mask']

                try:
                    validator_cache = clset['masking_validation']['cache']
                    validator_cache[(package_id, self.name, live)] = \
                        -1, myr
                except KeyError: # system settings client plugin not found
                    pass

                return -1, myr

    def _maskFilter_package_license_mask(self, package_id, live):

        if not self._settings['license_mask']:
            return

        mylicenses = self.retrieveLicense(package_id)
        mylicenses = mylicenses.strip().split()
        lic_mask = self._settings['license_mask']
        for mylicense in mylicenses:

            if mylicense not in lic_mask:
                continue

            ref = self._settings['pkg_masking_reference']
            myr = ref['user_license_mask']
            try:
                clset = self._client_settings
                validator_cache = clset['masking_validation']['cache']
                validator_cache[(package_id, self.name, live)] = -1, myr
            except KeyError: # system settings client plugin not found
                pass

            return -1, myr

    def _maskFilter_keyword_mask(self, package_id, live):

        # WORKAROUND for buggy entries
        # ** is fine then
        # TODO: remove this before 31-12-2011
        mykeywords = self.retrieveKeywords(package_id)
        if mykeywords == set([""]):
            mykeywords = set(['**'])

        mask_ref = self._settings['pkg_masking_reference']

        # firstly, check if package keywords are in etpConst['keywords']
        # (universal keywords have been merged from package.keywords)
        same_keywords = etpConst['keywords'] & mykeywords
        if same_keywords:
            myr = mask_ref['system_keyword']
            try:
                clset = self._client_settings
                validator_cache = clset['masking_validation']['cache']
                validator_cache[(package_id, self.name, live)] = \
                    package_id, myr

            except KeyError: # system settings client plugin not found
                pass

            return package_id, myr

        # if we get here, it means we didn't find mykeywords
        # in etpConst['keywords']
        # we need to seek self._settings['keywords']
        # seek in repository first
        keyword_repo = self._settings['keywords']['repositories']

        for keyword in tuple(keyword_repo.get(self.name, {}).keys()):

            if keyword not in mykeywords:
                continue

            keyword_data = keyword_repo[self.name].get(keyword)
            if not keyword_data:
                continue

            if "*" in keyword_data:
                # all packages in this repo with keyword "keyword" are ok
                myr = mask_ref['user_repo_package_keywords_all']
                try:
                    clset = self._client_settings
                    validator_cache = clset['masking_validation']['cache']
                    validator_cache[(package_id, self.name, live)] = \
                        package_id, myr
                except KeyError: # system settings client plugin not found
                    pass

                return package_id, myr

            kwd_key = "%s_ids" % (keyword,)
            keyword_data_ids = keyword_repo[self.name].get(kwd_key)
            if not isinstance(keyword_data_ids, set):

                keyword_data_ids = set()
                for atom in keyword_data:
                    matches, r = self.atomMatch(atom, multiMatch = True,
                        maskFilter = False)
                    if r != 0:
                        continue
                    keyword_data_ids |= matches

                keyword_repo[self.name][kwd_key] = keyword_data_ids

            if package_id in keyword_data_ids:

                myr = mask_ref['user_repo_package_keywords']
                try:
                    clset = self._client_settings
                    validator_cache = clset['masking_validation']['cache']
                    validator_cache[(package_id, self.name, live)] = \
                        package_id, myr
                except KeyError: # system settings client plugin not found
                    pass
                return package_id, myr

        keyword_pkg = self._settings['keywords']['packages']

        # if we get here, it means we didn't find a match in repositories
        # so we scan packages, last chance
        for keyword in keyword_pkg.keys():
            # use .keys() because keyword_pkg gets modified during iteration

            # first of all check if keyword is in mykeywords
            if keyword not in mykeywords:
                continue

            keyword_data = keyword_pkg.get(keyword)
            if not keyword_data:
                continue

            kwd_key = "%s_ids" % (keyword,)
            keyword_data_ids = keyword_pkg.get(self.name+kwd_key)

            if not isinstance(keyword_data_ids, (list, set)):
                keyword_data_ids = set()
                for atom in keyword_data:
                    # match atom
                    matches, r = self.atomMatch(atom, multiMatch = True,
                        maskFilter = False)
                    if r != 0:
                        continue
                    keyword_data_ids |= matches

                keyword_pkg[self.name+kwd_key] = keyword_data_ids

            if package_id in keyword_data_ids:

                # valid!
                myr = mask_ref['user_package_keywords']
                try:
                    clset = self._client_settings
                    validator_cache = clset['masking_validation']['cache']
                    validator_cache[(package_id, self.name, live)] = \
                        package_id, myr
                except KeyError: # system settings client plugin not found
                    pass

                return package_id, myr


        ## if we get here, it means that pkg it keyword masked
        ## and we should look at the very last resort, per-repository
        ## package keywords
        # check if repository contains keyword unmasking data

        clset = self._client_settings
        if clset is None:
            # SystemSettings Entropy Client plugin not available
            return

        # let's see if something is available in repository config
        repo_keywords = clset['repositories']['repos_keywords'].get(
            self.name)
        if repo_keywords is None:
            # nopers, sorry!
            return

        # check universal keywords
        same_keywords = repo_keywords.get('universal') & mykeywords
        if same_keywords:
            # universal keyword matches!
            myr = mask_ref['repository_packages_db_keywords']
            validator_cache = clset['masking_validation']['cache']
            validator_cache[(package_id, self.name, live)] = \
                package_id, myr
            return package_id, myr

        ## if we get here, it means that even universal masking failed
        ## and we need to look at per-package settings
        repo_settings = repo_keywords.get('packages')
        if not repo_settings:
            # it's empty, not worth checking
            return

        cached_key = "packages_ids"
        keyword_data_ids = repo_keywords.get(cached_key)
        if not isinstance(keyword_data_ids, dict):
            # create cache

            keyword_data_ids = {}
            for atom, values in repo_settings.items():
                matches, r = self.atomMatch(atom, multiMatch = True,
                    maskFilter = False)
                if r != 0:
                    continue
                for match in matches:
                    obj = keyword_data_ids.setdefault(match, set())
                    obj.update(values)

            repo_keywords[cached_key] = keyword_data_ids

        pkg_keywords = keyword_data_ids.get(package_id, set())
        if "**" in pkg_keywords:
            same_keywords = True
        else:
            same_keywords = pkg_keywords & etpConst['keywords']
        if same_keywords:
            # found! this pkg is not masked, yay!
            myr = mask_ref['repository_packages_db_keywords']
            validator_cache = clset['masking_validation']['cache']
            validator_cache[(package_id, self.name, live)] = \
                package_id, myr
            return package_id, myr

    def maskFilter(self, package_id, live = True):
        """
        Reimplemented from EntropyRepositoryBase
        """
        validator_cache = self._client_settings.get(
            'masking_validation', {}).get('cache', {})

        cached = validator_cache.get((package_id, self.name, live))
        if cached is not None:
            return cached

        # use on-disk cache?
        cached = self._mask_filter_fetch_cache(package_id)
        if cached is not None:
            return cached

        # avoid memleaks
        if len(validator_cache) > 100000:
            validator_cache.clear()

        if live:
            data = self._maskFilter_live(package_id)
            if data:
                return data

        data = self._maskFilter_user_package_mask(package_id, live)
        if data:
            self._mask_filter_store_cache(package_id, data)
            return data

        data = self._maskFilter_user_package_unmask(package_id, live)
        if data:
            self._mask_filter_store_cache(package_id, data)
            return data

        data = self._maskFilter_packages_db_mask(package_id, live)
        if data:
            self._mask_filter_store_cache(package_id, data)
            return data

        data = self._maskFilter_package_license_mask(package_id, live)
        if data:
            self._mask_filter_store_cache(package_id, data)
            return data

        data = self._maskFilter_keyword_mask(package_id, live)
        if data:
            self._mask_filter_store_cache(package_id, data)
            return data

        # holy crap, can't validate
        myr = self._settings['pkg_masking_reference']['completely_masked']
        validator_cache[(package_id, self.name, live)] = -1, myr
        self._mask_filter_store_cache(package_id, data)
        return -1, myr

    def atomMatchCacheKey(self):
        """
        Reimplemented from EntropyRepositoryBase.
        """
        return "%s_%s" % (
            SystemSettings().packages_configuration_hash(),
            self._settings_client_plugin.packages_configuration_hash(),
            )


class AvailablePackagesRepository(CachedRepository, MaskableRepository):
    """
    This class represents the available packages repository and is a direct
    subclass of EntropyRepository. It implements the update() method in order
    to make possible to update the repository.
    """
    def __init__(self, *args, **kwargs):
        super(AvailablePackagesRepository, self).__init__(*args, **kwargs)

        # ensure proper repository file permissions
        if entropy.tools.is_root() and os.path.isfile(self._db):
            const_setup_file(self._db, etpConst['entropygid'], 0o644,
                uid = etpConst['uid'])

    @staticmethod
    def update(entropy_client, repository_id, force, gpg):
        """
        Reimplemented from EntropyRepositoryBase
        """
        try:
            updater = AvailablePackagesRepositoryUpdater(
                entropy_client, repository_id,
                force, gpg)
        except KeyError as err:
            return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
        else:
            return updater.update()

    @staticmethod
    def revision(repository_id):
        """
        Reimplemented from EntropyRepositoryBase
        """
        db_data = SystemSettings()['repositories']['available'][repository_id]
        fname = os.path.join(db_data['dbpath'],
            etpConst['etpdatabaserevisionfile'])

        revision = -1
        enc = etpConst['conf_encoding']
        try:
            with codecs.open(fname, "r", encoding=enc) as f:
                read_data = f.readline().strip()
                try:
                    revision = int(read_data)
                except ValueError:
                    pass
        except (OSError, IOError) as err:
            if err.errno != errno.ENOENT:
                raise

        return revision

    @staticmethod
    def remote_revision(repository_id):
        """
        Reimplemented from EntropyRepositoryBase
        """
        return AvailablePackagesRepositoryUpdater(TextInterface(),
            repository_id, False, False).remote_revision()

    def handlePackage(self, pkg_data, revision = None,
                      formattedContent = False):
        """
        Reimplemented from EntropyRepository
        """
        raise PermissionDenied(
            "cannot execute handlePackage on this repository")

    def addPackage(self, pkg_data, revision = -1, package_id = None,
        formatted_content = False):
        """
        Reimplemented from EntropyRepository
        """
        raise PermissionDenied(
            "cannot execute addPackage on this repository")

    def removePackage(self, package_id, from_add_package = False):
        """
        Reimplemented from EntropyRepository
        """
        raise PermissionDenied(
            "cannot execute removePackage on this repository")

    def clearCache(self):
        # clear package masking filter
        try:
            clset = self._client_settings
        except AttributeError:
            # EntropyBaseRepository constructor calls
            # _maybeDatabaseSchemaUpdates that calls us here
            # while _real_client_settings is not set yet.
            pass
        else:
            clset.get('masking_validation', {}).get('cache', {}).clear()
        EntropyRepository.clearCache(self)


class GenericRepository(CachedRepository, MaskableRepository):
    """
    This class represents a generic packages repository and is a direct
    subclass of EntropyRepository.
    Even GenericRepository is a CachedRepository because its object could
    get cached by 3rd party. Actually, we require this because our installed
    packages repository could end up being a GenericRepository, when running
    in fail-safe mode.
    """

    def handlePackage(self, pkg_data, revision = None,
                      formattedContent = False):
        """
        Reimplemented from EntropyRepository.
        It is supposed that a generic repository should not support
        handlePackage. You can override this (at your own risk) by setting the
        "override_handlePackage" property to True. In this case, a generic
        addPackage() call is issued.
        """
        override = getattr(self, 'override_handlePackage', False)
        if not override:
            raise PermissionDenied(
                "cannot execute handlePackage on this repository")

        return self.addPackage(pkg_data, revision = revision,
                               formatted_content = formattedContent)

    def maskFilter(self, package_id, live = True):
        """
        Reimplemented from EntropyRepository.
        It is supposed that a generic repository doesn't support package
        masking. You can override this by setting the "enable_mask_filter"
        to True.
        """
        enabled = getattr(self, 'enable_mask_filter', False)
        if not enabled:
            return package_id, 0
        return MaskableRepository.maskFilter(self, package_id, live = live)
