Edit - files.py

Edit: files.py

# -*- coding: utf-8 -*-

from __future__ import print_function, division, absolute_import
#
# Copyright (c) 2018 Red Hat, Inc.
#
# This software is licensed to you under the GNU General Public License,
# version 2 (GPLv2). There is NO WARRANTY for this software, express or
# implied, including the implied warranties of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
# along with this software; if not, see
# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
#
# Red Hat trademarks are not licensed under GPLv2. No permission is
# granted to use or replicate Red Hat trademarks that are incorporated
# in this software or its documentation.

"""
This module contains utilities for manipulating files pertaining to system syspurpose
"""

import collections
import logging
import json
import os
import errno
import io
from syspurpose.utils import system_exit, create_dir, create_file, make_utf8, write_to_file_utf8
from syspurpose.i18n import ugettext as _

# Constants for locations of the two system syspurpose files
USER_SYSPURPOSE_DIR = "/etc/rhsm/syspurpose"
USER_SYSPURPOSE = os.path.join(USER_SYSPURPOSE_DIR, "syspurpose.json")
VALID_FIELDS = os.path.join(USER_SYSPURPOSE_DIR, "valid_fields.json")  # Will be used for future validation
CACHE_DIR = "/var/lib/rhsm/cache"
CACHED_SYSPURPOSE = os.path.join(CACHE_DIR, "syspurpose.json")  # Stores cached values

# All names that represent syspurpose values locally
ROLE = 'role'
ADDONS = 'addons'
SERVICE_LEVEL = 'service_level_agreement'
USAGE = 'usage'

# Remote values keyed on the local ones
LOCAL_TO_REMOTE = {
    ROLE: 'role',
    ADDONS: 'addOns',
    SERVICE_LEVEL: 'serviceLevel',
    USAGE: 'usage'
}

# All known syspurpose attributes
ATTRIBUTES = [ROLE, ADDONS, SERVICE_LEVEL, USAGE]

# Values used in determining changes between client and server
UNSUPPORTED = "unsupported"

log = logging.getLogger(__name__)

def post_process_received_data(data):
    """
    Try to solve conflicts in keys
     - Server returns key "roles", but it should be "role"
     - Server returns key "support_level", but service_level_agreement is used in syspurpose.json
    :return: modified dictionary
    """
    if 'systemPurposeAttributes' in data:
        # Fix
        if 'roles' in data['systemPurposeAttributes']:
            data['systemPurposeAttributes']['role'] = data['systemPurposeAttributes']['roles']
            del data['systemPurposeAttributes']['roles']
        if 'support_level' in data['systemPurposeAttributes']:
            data['systemPurposeAttributes']['service_level_agreement'] = data['systemPurposeAttributes']['support_level']
            del data['systemPurposeAttributes']['support_level']
    return data

class SyspurposeStore(object):
    """
    Represents and maintains a json syspurpose file
    """

def __init__(self, path, raise_on_error=False):
        self.path = path
        self.contents = {}
        self.raise_on_error = raise_on_error

def read_file(self):
        """
        Opens & reads the contents of the store's file based on the 'path' provided to the constructor,
        and stores them on this object. If the user doesn't have access rights to the file, the program exits.
        :return: False if the contents of the file were empty, or the file doesn't exist; otherwise, nothing.
        """
        try:
            with io.open(self.path, 'r', encoding='utf-8') as f:
                self.contents = json.load(f)
                return True
        except ValueError:
            # Malformed JSON or empty file. Let's not error out on an empty file
            if os.path.getsize(self.path):
                system_exit(
                    os.EX_CONFIG,
                    _("Error: Malformed data in file {}; please review and correct.").format(self.path)
                )

return False
        except OSError as e:
            if e.errno == errno.EACCES and not self.raise_on_error:
                system_exit(os.EX_NOPERM, _('Cannot read syspurpose file {}\nAre you root?').format(self.path))
            if e.errno == errno.ENOENT and not self.raise_on_error:
                log.error('Unable to read file {file}: {error}'.format(file=self.path, error=e))
                return False
            if self.raise_on_error:
                raise e

def create(self):
        """
        Create the files necessary for this store
        :return: True if changes were made, false otherwise
        """
        return create_dir(os.path.dirname(self.path)) or \
            self.read_file() or \
            create_file(self.path, self.contents)

def add(self, key, value):
        """
        Add a value to a list of values specified by key. If the current value specified by the key is scalar/non-list,
        it is not overridden, but maintained in the list, along with the new value.
        :param key: The name of the list
        :param value: The value to append to the list
        :return: None
        """
        value = make_utf8(value)
        key = make_utf8(key)
        try:
            current_value = self.contents[key]
            if current_value is not None and not isinstance(current_value, list):
                self.contents[key] = [current_value]

if self.contents[key] is None:
                self.contents[key] = []

if value not in self.contents[key]:
                self.contents[key].append(value)
            else:
                return False
        except (AttributeError, KeyError):
            self.contents[key] = [value]
        return True

def remove(self, key, value):
        """
        Remove a value from a list specified by key.
        If the current value specified by the key is not a list, unset the value.
        :param key: The name of the list parameter to manipulate
        :param value: The value to attempt to remove
        :return: True if the value was in the list, False if it was not
        """
        value = make_utf8(value)
        key = make_utf8(key)
        try:
            current_value = self.contents[key]
            if current_value is not None and not isinstance(current_value, list) and current_value == value:
                return self.unset(key)

if value in current_value:
                self.contents[key].remove(value)
            else:
                return False

return True
        except (AttributeError, KeyError, ValueError):
            return False

def unset(self, key):
        """
        Unsets a key
        :param key: The key to unset
        :return: boolean
        """
        key = make_utf8(key)

# Special handling is required for the SLA, since it deviates from the typical CP
        # empty => null semantics
        if key == 'service_level_agreement':
            value = self.contents.get(key, None)
            self.contents[key] = ''
        else:
            value = self.contents.pop(key, None)

return value is not None

def set(self, key, value):
        """
        Set a key (syspurpose parameter) to value
        :param key: The parameter of the syspurpose file to set
        :type key: str

:param value: The value to set that parameter to
        :return: Whether any change was made
        """
        value = make_utf8(value)
        key = make_utf8(key)
        org = make_utf8(self.contents.get(key, None))
        self.contents[key] = value
        return org != value or org is None

def write(self, fp=None):
        """
        Write the current contents to the file at self.path
        """
        if not fp:
            with io.open(self.path, 'w', encoding='utf-8') as f:
                write_to_file_utf8(f, self.contents)
                f.flush()
        else:
            write_to_file_utf8(fp, self.contents)

@classmethod
    def read(cls, path, raise_on_error=False):
        """
        Read the file represented by path. If the file does not exist it is created.
        :param path: The path on the file system to read, should be a json file
        :param raise_on_error: When it is set to True, then exceptions are raised as expected.
        :return: new SyspurposeStore with the contents read in
        """
        new_store = cls(path, raise_on_error=raise_on_error)

if not os.access(path, os.W_OK):
            new_store.create()
        else:
            new_store.read_file()

return new_store

class SyncResult(object):
    """
    A container class for the results of a sync operation performed by a SyncedStore class.
    """

def __init__(self, result, remote_changed, local_changed, cached_changed):
        self.result = result
        self.remote_changed = remote_changed
        self.local_changed = local_changed
        self.cached_changed = cached_changed

class SyncedStore(object):
    """
    Stores values in a local file backed by a cache which is then synced with another source
    of the same values.
    """
    PATH = USER_SYSPURPOSE
    CACHE_PATH = CACHED_SYSPURPOSE

def __init__(self, uep, on_changed=None, consumer_uuid=None, use_valid_fields=False):
        """
        Initialization of SyncedStore
        :param uep: object representing connection to candlepin server
        :param on_changed: optional callback method called, during three-way merge
        :param consumer_uuid: UUID of consumer
        :param use_valid_fields: if valid fields are considered
        """
        self.uep = uep
        self.filename = self.PATH.split('/')[-1]
        self.path = self.PATH
        self.cache_path = self.CACHE_PATH
        self.local_file = None
        self.local_contents = self.get_local_contents()
        self.cache_file = None
        self.cache_contents = self.get_cached_contents()
        self.changed = False
        self.on_changed = on_changed
        self.consumer_uuid = consumer_uuid
        if use_valid_fields is True:
            self.valid_fields = self.get_valid_fields()
        else:
            self.valid_fields = None

def __enter__(self):
        return self

def __exit__(self, exc_type, exc_val, exc_tb):
        self.finish()

def finish(self):
        """
        When local content was changed, then try to synchronize local content with remote server
        :return:
        """
        if self.changed:
            self.sync()

def sync(self):
        """
        Try to synchronize local content with remote server
        :return: instance of SyncResult holding result of synchronization
        """
        log.debug('Attempting to sync syspurpose content...')
        try:
            if self.uep and not self.uep.has_capability('syspurpose'):
                log.debug('Server does not support syspurpose, syncing only locally.')
                return self._sync_local_only()
        except Exception as err:
            log.debug(
                'Failed to detect whether the server has syspurpose capability: {err}'.format(
                    err=err
                )
            )
            return self._sync_local_only()

remote_contents = self.get_remote_contents()
        local_contents = self.get_local_contents()
        cached_contents = self.get_cached_contents()

result = self.merge(local=local_contents,
                            remote=remote_contents,
                            base=cached_contents)

local_result = {key: result[key] for key in result if result[key]}

sync_result = SyncResult(
            result,
            (remote_contents == result) or self.update_remote(result),
            self.update_local(local_result),
            self.update_cache(result),
        )

log.debug('Successfully synced system purpose.')

# Reset the changed attribute as all items should be synced if we've gotten to this point
        self.changed = False

return sync_result

def _sync_local_only(self):
        local_updated = self.update_local(self.get_local_contents())
        return SyncResult(self.local_contents, False, local_updated, False)

def merge(self, local=None, remote=None, base=None):
        """
        Do three-way merge
        :param local: dictionary with local values (syspyrpose.json)
        :param remote: dictionary with values from server
        :param base:
        :return:
        """
        result = three_way_merge(
            local=local,
            base=base,
            remote=remote,
            on_change=self.on_changed
        )
        return result

def get_local_contents(self):
        """
        Try to load local content from file
        :return: dictionary with system purpose values
        """
        try:
            self.local_contents = json.load(io.open(self.path, 'r', encoding='utf-8'))
        except (os.error, ValueError, IOError):
            log.debug('Unable to read local system purpose at "%s"' % self.path)
            self.update_local({})
            self.local_contents = {}
        return self.local_contents

def get_remote_contents(self):
        """
        Try to get remote content from server
        :return: dictionary with system purpose values
        """
        if self.uep is None or self.consumer_uuid is None:
            log.debug('Failed to read remote syspurpose from server: no available connection, '
                      'or the consumer is not registered.')
            return {}
        if not self.uep.has_capability('syspurpose'):
            log.debug('Server does not support syspurpose, not syncing.')
            return {}

consumer = self.uep.getConsumer(self.consumer_uuid)
        result = {}

# Translate from the remote values to the local, filtering out items not known
        for attr in ATTRIBUTES:
            value = consumer.get(LOCAL_TO_REMOTE[attr])
            result[attr] = value
        log.debug('Successfully read remote syspurpose from server.')

return result

def get_cached_contents(self):
        """
        Try to load cached server response from the file
        :return: dictionary with system purpose values
        """
        try:
            self.cache_contents = json.load(io.open(self.cache_path, 'r', encoding='utf-8'))
            log.debug('Successfully read cached syspurpose contents.')
        except (ValueError, os.error, IOError):
            log.debug('Unable to read cached syspurpose contents at \'%s\'.' % self.path)
            self.cache_contents = {}
            self.update_cache({})
        return self.cache_contents

def update_local(self, data):
        """
        Rewrite local content with new data and write data to file syspurpose.json
        :param data: new dictionary with local data
        :return: None
        """
        self.local_contents = data
        self._write_local()

def _write_local(self):
        """
        Write local data to the file
        :return: None
        """
        self._update_file(self.path, self.local_contents)

def update_cache(self, data):
        self.cache_contents = data
        self._write_cache()

def _write_cache(self):
        """
        Write cache to file
        :return: None
        """
        self._update_file(self.cache_path, self.cache_contents)

def update_remote(self, data):
        if self.uep is None or self.consumer_uuid is None:
            log.debug('Failed to update remote syspurpose on the server: no available connection, '
                      'or the consumer is not registered.')
            return False

addons = data.get(ADDONS)
        self.uep.updateConsumer(
                self.consumer_uuid,
                role=data.get(ROLE) or "",
                addons=addons if addons is not None else [],
                service_level=data.get(SERVICE_LEVEL) or "",
                usage=data.get(USAGE) or ""
        )
        log.debug('Successfully updated remote syspurpose on the server.')
        return True

def _check_key_value_validity(self, key, value):
        """
        Check validity of provided key and value of it is included in valid fields
        :param key: provided key
        :param value: provided value
        :return: None
        """
        if self.valid_fields is not None:
            if key in self.valid_fields:
                if value not in self.valid_fields[key]:
                    print(
                        _('Warning: Provided value "{val}" is not included in the list '
                          'of valid values for attribute {attr}:').format(val=value, attr=key)
                    )
                    for valid_value in self.valid_fields[key]:
                        if len(valid_value) > 0:
                            print(" - %s" % valid_value)
            else:
                print(_('Warning: Provided key "{key}" is not included in the list of valid keys:').format(
                    key=key
                ))
                for valid_key in self.valid_fields.keys():
                    print(" - %s" % valid_key)

def add(self, key, value):
        """
        Add a value to a list of values specified by key. If the current value specified by the key is scalar/non-list,
        it is not overridden, but maintained in the list, along with the new value.
        :param key: The name of the list
        :param value: The value to append to the list
        :return: None
        """
        value = make_utf8(value)
        key = make_utf8(key)
        try:
            # When existing value was set using set() method, then the
            # existing valus is not list, but simple value. We have to convert
            # it first
            current_value = self.local_contents[key]
            if current_value is not None and not isinstance(current_value, list):
                self.local_contents[key] = [current_value]

# When existing value is None, then first covert to empty list to be
            # able to call append method. It is very theoretical case.
            if self.local_contents[key] is None:
                self.local_contents[key] = []

if value not in self.local_contents[key]:
                self.local_contents[key].append(value)
            else:
                log.debug('Will not add value \'%s\' to key \'%s\'.' % (value, key))
                self.changed = False
                return self.changed
        except (AttributeError, KeyError):
            self.local_contents[key] = [value]

self._check_key_value_validity(key, value)

self.changed = True
        log.debug('Adding value \'%s\' to key \'%s\'.' % (value, key))

# Write changes to the syspurpose.json file
        if self.changed is True:
            self._write_local()

return self.changed

def remove(self, key, value):
        """
        Remove a value from a list specified by key.
        If the current value specified by the key is not a list, unset the value.
        :param key: The name of the list parameter to manipulate
        :param value: The value to attempt to remove
        :return: True if the value was in the list, False if it was not
        """
        value = make_utf8(value)
        key = make_utf8(key)
        try:
            current_values = self.local_contents[key]
            if current_values is not None and not isinstance(current_values, list) and current_values == value:
                return self.unset(key)

if value in current_values:
                self.local_contents[key].remove(value)
                self.changed = True
                log.debug('Removing value \'%s\' from key \'%s\'.' % (value, key))
            else:
                self.changed = False
                log.debug('Will not remove value \'%s\' from key \'%s\'.' % (value, key))
                return self.changed

except (AttributeError, KeyError, ValueError):
            log.debug('Will not remove value \'%s\' from key \'%s\'.' % (value, key))
            self.changed = False

# Write changes to the syspurpose.json file
        if self.changed is True:
            self._write_local()

return self.changed

def unset(self, key):
        """
        Unsets a key
        :param key: The key to unset
        :return: boolean
        """
        key = make_utf8(key)

# Special handling is required for the SLA, since it deviates from the typical CP
        # empty => null semantics
        if key == 'service_level_agreement':
            value = self.local_contents.get(key, None)
            self.local_contents[key] = ''
        elif key == 'addons':
            value = self.local_contents.get(key, None)
            self.local_contents[key] = []
        else:
            value = self.local_contents.pop(key, None)
        self.changed = True
        log.debug('Unsetting value \'%s\' of key \'%s\'.' % (value, key))

self.changed = value is not None

# Write changes to the syspurpose.json file
        if self.changed is True:
            self._write_local()

return self.changed

def set(self, key, value):
        """
        Set a key (syspurpose parameter) to value
        :param key: The parameter of the syspurpose file to set
        :type key: str

:param value: The value to set that parameter to
        :return: Whether any change was made
        """
        value = make_utf8(value)
        key = make_utf8(key)
        current_value = make_utf8(self.local_contents.get(key, None))
        self.local_contents[key] = value

if current_value != value or current_value is None:
            self._check_key_value_validity(key, value)

self.changed = True
            log.debug('Setting value \'%s\' to key \'%s\'.' % (value, key))
        else:
            log.debug('NOT Setting value \'%s\' to key \'%s\'.')

self.changed = current_value != value or current_value is None

# Write changes to the syspurpose.json file
        if self.changed is True:
            self._write_local()

return self.changed

@staticmethod
    def _create_missing_dir(dir_path):
        """
        Try to create missing directory
        :param dir_path: path to directory
        :return: None
        """
        # Check if the directory exists
        if not os.path.isdir(dir_path):
            log.debug('Trying to create directory: %s' % dir_path)
            try:
                os.makedirs(dir_path, mode=0o755, exist_ok=True)
            except Exception as err:
                log.warning('Unable to create directory: %s, error: %s' % (dir_path, err))

@classmethod
    def _update_file(cls, path, data):
        """
        Write the contents of data to file in the first mode we can (effectively to create or update
        the file)
        :param path: The string path to the file location we should update
        :param data: The data to write to the file
        :return: None
        """

# Check if /etc/rhsm/syspurpose directory exists
        cls._create_missing_dir(USER_SYSPURPOSE_DIR)
        # Check if /var/lib/rhsm/cache/ directory exists
        cls._create_missing_dir(CACHE_DIR)

# Then we can try to create syspurpose.json file
        try:
            f = io.open(path, 'w+', encoding='utf-8')
        except OSError as e:
            if e.errno != 17:
                raise
        else:
            write_to_file_utf8(f, data)
            f.flush()
            f.close()
            log.debug('Successfully updated syspurpose values at \'%s\'.' % path)
        log.debug('Failed to update syspurpose values at \'%s\'.' % path)

def get_valid_fields(self):
        """
        Try to get valid fields from server using current owner (organization)
        :return: Dictionary with valid fields
        """
        valid_fields = None

if self.uep is not None and self.consumer_uuid is not None:
            current_owner = self.uep.getOwner(self.consumer_uuid)
            if 'key' in current_owner:
                owner_key = current_owner['key']
                try:
                    response = self.uep.getOwnerSyspurposeValidFields(owner_key)
                except Exception as err:
                    log.debug("Unable to get valid fields from server: %s" % err)
                else:
                    if 'systemPurposeAttributes' in response:
                        response = post_process_received_data(response)
                        valid_fields = response['systemPurposeAttributes']
        return valid_fields

# A simple container class used to hold the values representing a change detected
# during three_way_merge
DiffChange = collections.namedtuple(
    'DiffChange',
    ['key', 'previous_value', 'new_value', 'source', 'in_base', 'in_result']
)

def three_way_merge(local, base, remote, on_conflict="remote", on_change=None):
    """
    Performs a three-way merge on the local and remote dictionaries with a given base.
    :param local: The dictionary of the current local values
    :param base: The dictionary with the values we've last seen
    :param remote: The dictionary with "their" values
    :param on_conflict: Either "remote" or "local" or None. If "remote", the remote changes
                               will win any conflict. If "local", the local changes will win any
                               conflict. If anything else, an error will be thrown.
    :param on_change: This is an optional function which will be given each change as it is
                      detected.
    :return: The dictionary of values as merged between the three provided dictionaries.
    """
    log.debug('Attempting a three-way merge...')
    result = {}
    local = local or {}
    base = base or {}
    remote = remote or {}

if on_conflict == "remote":
        winner = remote
    elif on_conflict == "local":
        winner = local
    else:
        raise ValueError('keyword argument "on_conflict" must be either "remote" or "local"')

if on_change is None:
        on_change = lambda change: change

all_keys = set(local.keys()) | set(base.keys()) | set(remote.keys())

for key in all_keys:

local_changed = detect_changed(base=base, other=local, key=key, source="local")
        remote_changed = detect_changed(base=base, other=remote, key=key, source="server")
        changed = local_changed or remote_changed and remote_changed != UNSUPPORTED
        source = 'base'

if local_changed == remote_changed:
            if local_changed is True:
                log.debug('Three way merge conflict: both local and remote values changed for key \'%s\'.' % key)
            source = on_conflict
            if key in winner:
                result[key] = winner[key]
        elif remote_changed is True:
            log.debug('Three way merge: remote value was changed for key \'%s\'.' % key)
            source = 'remote'
            if key in remote:
                result[key] = remote[key]
        elif local_changed or remote_changed == UNSUPPORTED:
            if local_changed is True:
                log.debug('Three way merge: local value was changed for key \'%s\'.' % key)
            source = 'local'
            if key in local:
                result[key] = local[key]

if changed:
            original = base.get(key)
            diff = DiffChange(key=key, source=source, previous_value=original,
                              new_value=result.get(key), in_base=key in base,
                              in_result=key in result)
            on_change(diff)

return result

def detect_changed(base, other, key, source="server"):
    """
    Detect the type of change that has occurred between base and other for a given key.
    :param base: The dictionary of values we are starting with
    :param other: The dictionary of now current values
    :param key: The key that we are interested in knowing how it changed
    :param source: An optional string which indicates where the "other" values came from. Used to
                   make decisions which are one sided. (i.e. only applicable for changes from the
                   server side).
    :return: True if there was a change, false if there was no change
    :rtype: bool
    """
    base = base or {}
    other = other or {}
    if key not in other and source != "local":
        return UNSUPPORTED

base_val = base.get(key)
    other_val = other.get(key)

if key not in other and source == "local":
        # If the local values no longer contain the key we want to treat this as removal
        # It would constitute a change if the base had a truthy value. The values tracked from the
        # server all have falsey values.
        return bool(base_val)

# Handle "addons" (the lists might be out of order from the server)
    if type(base_val) == list and type(other_val) == list:
        return sorted(base_val) != sorted(other_val)

# When value is removed from server, then it is set to empty string, but
    # it is completely removed from local syspurpose.json.
    # See: https://bugzilla.redhat.com/show_bug.cgi?id=1738764
    if source == "server" and base_val is None and other_val == '':
        return False

return base_val != other_val

Cancel