Edit: frontend.py
# -*- coding: utf-8 -*- """ babel.messages.frontend ~~~~~~~~~~~~~~~~~~~~~~~ Frontends for the message extraction functionality. :copyright: (c) 2013-2021 by the Babel Team. :license: BSD, see LICENSE for more details. """ from __future__ import print_function import logging import optparse import os import re import shutil import sys import tempfile from collections import OrderedDict from datetime import datetime from locale import getpreferredencoding from babel import __version__ as VERSION from babel import Locale, localedata from babel._compat import StringIO, string_types, text_type, PY2 from babel.core import UnknownLocaleError from babel.messages.catalog import Catalog from babel.messages.extract import DEFAULT_KEYWORDS, DEFAULT_MAPPING, check_and_call_extract_file, extract_from_dir from babel.messages.mofile import write_mo from babel.messages.pofile import read_po, write_po from babel.util import LOCALTZ from distutils import log as distutils_log from distutils.cmd import Command as _Command from distutils.errors import DistutilsOptionError, DistutilsSetupError try: from ConfigParser import RawConfigParser except ImportError: from configparser import RawConfigParser po_file_read_mode = ('rU' if PY2 else 'r') def listify_value(arg, split=None): """ Make a list out of an argument. Values from `distutils` argument parsing are always single strings; values from `optparse` parsing may be lists of strings that may need to be further split. No matter the input, this function returns a flat list of whitespace-trimmed strings, with `None` values filtered out. >>> listify_value("foo bar") ['foo', 'bar'] >>> listify_value(["foo bar"]) ['foo', 'bar'] >>> listify_value([["foo"], "bar"]) ['foo', 'bar'] >>> listify_value([["foo"], ["bar", None, "foo"]]) ['foo', 'bar', 'foo'] >>> listify_value("foo, bar, quux", ",") ['foo', 'bar', 'quux'] :param arg: A string or a list of strings :param split: The argument to pass to `str.split()`. :return: """ out = [] if not isinstance(arg, (list, tuple)): arg = [arg] for val in arg: if val is None: continue if isinstance(val, (list, tuple)): out.extend(listify_value(val, split=split)) continue out.extend(s.strip() for s in text_type(val).split(split)) assert all(isinstance(val, string_types) for val in out) return out class Command(_Command): # This class is a small shim between Distutils commands and # optparse option parsing in the frontend command line. #: Option name to be input as `args` on the script command line. as_args = None #: Options which allow multiple values. #: This is used by the `optparse` transmogrification code. multiple_value_options = () #: Options which are booleans. #: This is used by the `optparse` transmogrification code. # (This is actually used by distutils code too, but is never # declared in the base class.) boolean_options = () #: Option aliases, to retain standalone command compatibility. #: Distutils does not support option aliases, but optparse does. #: This maps the distutils argument name to an iterable of aliases #: that are usable with optparse. option_aliases = {} #: Choices for options that needed to be restricted to specific #: list of choices. option_choices = {} #: Log object. To allow replacement in the script command line runner. log = distutils_log def __init__(self, dist=None): # A less strict version of distutils' `__init__`. self.distribution = dist self.initialize_options() self._dry_run = None self.verbose = False self.force = None self.help = 0 self.finalized = 0 class compile_catalog(Command): """Catalog compilation command for use in ``setup.py`` scripts. If correctly installed, this command is available to Setuptools-using setup scripts automatically. For projects using plain old ``distutils``, the command needs to be registered explicitly in ``setup.py``:: from babel.messages.frontend import compile_catalog setup( ... cmdclass = {'compile_catalog': compile_catalog} ) .. versionadded:: 0.9 """ description = 'compile message catalogs to binary MO files' user_options = [ ('domain=', 'D', "domains of PO files (space separated list, default 'messages')"), ('directory=', 'd', 'path to base directory containing the catalogs'), ('input-file=', 'i', 'name of the input file'), ('output-file=', 'o', "name of the output file (default " "'<output_dir>/<locale>/LC_MESSAGES/<domain>.mo')"), ('locale=', 'l', 'locale of the catalog to compile'), ('use-fuzzy', 'f', 'also include fuzzy translations'), ('statistics', None, 'print statistics about translations') ] boolean_options = ['use-fuzzy', 'statistics'] def initialize_options(self): self.domain = 'messages' self.directory = None self.input_file = None self.output_file = None self.locale = None self.use_fuzzy = False self.statistics = False def finalize_options(self): self.domain = listify_value(self.domain) if not self.input_file and not self.directory: raise DistutilsOptionError('you must specify either the input file ' 'or the base directory') if not self.output_file and not self.directory: raise DistutilsOptionError('you must specify either the output file ' 'or the base directory') def run(self): n_errors = 0 for domain in self.domain: for catalog, errors in self._run_domain(domain).items(): n_errors += len(errors) if n_errors: self.log.error('%d errors encountered.' % n_errors) return (1 if n_errors else 0) def _run_domain(self, domain): po_files = [] mo_files = [] if not self.input_file: if self.locale: po_files.append((self.locale, os.path.join(self.directory, self.locale, 'LC_MESSAGES', domain + '.po'))) mo_files.append(os.path.join(self.directory, self.locale, 'LC_MESSAGES', domain + '.mo')) else: for locale in os.listdir(self.directory): po_file = os.path.join(self.directory, locale, 'LC_MESSAGES', domain + '.po') if os.path.exists(po_file): po_files.append((locale, po_file)) mo_files.append(os.path.join(self.directory, locale, 'LC_MESSAGES', domain + '.mo')) else: po_files.append((self.locale, self.input_file)) if self.output_file: mo_files.append(self.output_file) else: mo_files.append(os.path.join(self.directory, self.locale, 'LC_MESSAGES', domain + '.mo')) if not po_files: raise DistutilsOptionError('no message catalogs found') catalogs_and_errors = {} for idx, (locale, po_file) in enumerate(po_files): mo_file = mo_files[idx] with open(po_file, 'rb') as infile: catalog = read_po(infile, locale) if self.statistics: translated = 0 for message in list(catalog)[1:]: if message.string: translated += 1 percentage = 0 if len(catalog): percentage = translated * 100 // len(catalog) self.log.info( '%d of %d messages (%d%%) translated in %s', translated, len(catalog), percentage, po_file ) if catalog.fuzzy and not self.use_fuzzy: self.log.info('catalog %s is marked as fuzzy, skipping', po_file) continue catalogs_and_errors[catalog] = catalog_errors = list(catalog.check()) for message, errors in catalog_errors: for error in errors: self.log.error( 'error: %s:%d: %s', po_file, message.lineno, error ) self.log.info('compiling catalog %s to %s', po_file, mo_file) with open(mo_file, 'wb') as outfile: write_mo(outfile, catalog, use_fuzzy=self.use_fuzzy) return catalogs_and_errors class extract_messages(Command): """Message extraction command for use in ``setup.py`` scripts. If correctly installed, this command is available to Setuptools-using setup scripts automatically. For projects using plain old ``distutils``, the command needs to be registered explicitly in ``setup.py``:: from babel.messages.frontend import extract_messages setup( ... cmdclass = {'extract_messages': extract_messages} ) """ description = 'extract localizable strings from the project code' user_options = [ ('charset=', None, 'charset to use in the output file (default "utf-8")'), ('keywords=', 'k', 'space-separated list of keywords to look for in addition to the ' 'defaults (may be repeated multiple times)'), ('no-default-keywords', None, 'do not include the default keywords'), ('mapping-file=', 'F', 'path to the mapping configuration file'), ('no-location', None, 'do not include location comments with filename and line number'), ('add-location=', None, 'location lines format. If it is not given or "full", it generates ' 'the lines with both file name and line number. If it is "file", ' 'the line number part is omitted. If it is "never", it completely ' 'suppresses the lines (same as --no-location).'), ('omit-header', None, 'do not include msgid "" entry in header'), ('output-file=', 'o', 'name of the output file'), ('width=', 'w', 'set output line width (default 76)'), ('no-wrap', None, 'do not break long message lines, longer than the output line width, ' 'into several lines'), ('sort-output', None, 'generate sorted output (default False)'), ('sort-by-file', None, 'sort output by file location (default False)'), ('msgid-bugs-address=', None, 'set report address for msgid'), ('copyright-holder=', None, 'set copyright holder in output'), ('project=', None, 'set project name in output'), ('version=', None, 'set project version in output'), ('add-comments=', 'c', 'place comment block with TAG (or those preceding keyword lines) in ' 'output file. Separate multiple TAGs with commas(,)'), # TODO: Support repetition of this argument ('strip-comments', 's', 'strip the comment TAGs from the comments.'), ('input-paths=', None, 'files or directories that should be scanned for messages. Separate multiple ' 'files or directories with commas(,)'), # TODO: Support repetition of this argument ('input-dirs=', None, # TODO (3.x): Remove me. 'alias for input-paths (does allow files as well as directories).'), ] boolean_options = [ 'no-default-keywords', 'no-location', 'omit-header', 'no-wrap', 'sort-output', 'sort-by-file', 'strip-comments' ] as_args = 'input-paths' multiple_value_options = ('add-comments', 'keywords') option_aliases = { 'keywords': ('--keyword',), 'mapping-file': ('--mapping',), 'output-file': ('--output',), 'strip-comments': ('--strip-comment-tags',), } option_choices = { 'add-location': ('full', 'file', 'never',), } def initialize_options(self): self.charset = 'utf-8' self.keywords = None self.no_default_keywords = False self.mapping_file = None self.no_location = False self.add_location = None self.omit_header = False self.output_file = None self.input_dirs = None self.input_paths = None self.width = None self.no_wrap = False self.sort_output = False self.sort_by_file = False self.msgid_bugs_address = None self.copyright_holder = None self.project = None self.version = None self.add_comments = None self.strip_comments = False self.include_lineno = True def finalize_options(self): if self.input_dirs: if not self.input_paths: self.input_paths = self.input_dirs else: raise DistutilsOptionError( 'input-dirs and input-paths are mutually exclusive' ) if self.no_default_keywords: keywords = {} else: keywords = DEFAULT_KEYWORDS.copy() keywords.update(parse_keywords(listify_value(self.keywords))) self.keywords = keywords if not self.keywords: raise DistutilsOptionError('you must specify new keywords if you ' 'disable the default ones') if not self.output_file: raise DistutilsOptionError('no output file specified') if self.no_wrap and self.width: raise DistutilsOptionError("'--no-wrap' and '--width' are mutually " "exclusive") if not self.no_wrap and not self.width: self.width = 76 elif self.width is not None: self.width = int(self.width) if self.sort_output and self.sort_by_file: raise DistutilsOptionError("'--sort-output' and '--sort-by-file' " "are mutually exclusive") if self.input_paths: if isinstance(self.input_paths, string_types): self.input_paths = re.split(r',\s*', self.input_paths) elif self.distribution is not None: self.input_paths = dict.fromkeys([ k.split('.', 1)[0] for k in (self.distribution.packages or ()) ]).keys() else: self.input_paths = [] if not self.input_paths: raise DistutilsOptionError("no input files or directories specified") for path in self.input_paths: if not os.path.exists(path): raise DistutilsOptionError("Input path: %s does not exist" % path) self.add_comments = listify_value(self.add_comments or (), ",") if self.distribution: if not self.project: self.project = self.distribution.get_name() if not self.version: self.version = self.distribution.get_version() if self.add_location == 'never': self.no_location = True elif self.add_location == 'file': self.include_lineno = False def run(self): mappings = self._get_mappings() with open(self.output_file, 'wb') as outfile: catalog = Catalog(project=self.project, version=self.version, msgid_bugs_address=self.msgid_bugs_address, copyright_holder=self.copyright_holder, charset=self.charset) for path, method_map, options_map in mappings: def callback(filename, method, options): if method == 'ignore': return # If we explicitly provide a full filepath, just use that. # Otherwise, path will be the directory path and filename # is the relative path from that dir to the file. # So we can join those to get the full filepath. if os.path.isfile(path): filepath = path else: filepath = os.path.normpath(os.path.join(path, filename)) optstr = '' if options: optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for k, v in options.items()]) self.log.info('extracting messages from %s%s', filepath, optstr) if os.path.isfile(path): current_dir = os.getcwd() extracted = check_and_call_extract_file( path, method_map, options_map, callback, self.keywords, self.add_comments, self.strip_comments, current_dir ) else: extracted = extract_from_dir( path, method_map, options_map, keywords=self.keywords, comment_tags=self.add_comments, callback=callback, strip_comment_tags=self.strip_comments ) for filename, lineno, message, comments, context in extracted: if os.path.isfile(path): filepath = filename # already normalized else: filepath = os.path.normpath(os.path.join(path, filename)) catalog.add(message, None, [(filepath, lineno)], auto_comments=comments, context=context) self.log.info('writing PO template file to %s', self.output_file) write_po(outfile, catalog, width=self.width, no_location=self.no_location, omit_header=self.omit_header, sort_output=self.sort_output, sort_by_file=self.sort_by_file, include_lineno=self.include_lineno) def _get_mappings(self): mappings = [] if self.mapping_file: with open(self.mapping_file, po_file_read_mode) as fileobj: method_map, options_map = parse_mapping(fileobj) for path in self.input_paths: mappings.append((path, method_map, options_map)) elif getattr(self.distribution, 'message_extractors', None): message_extractors = self.distribution.message_extractors for path, mapping in message_extractors.items(): if isinstance(mapping, string_types): method_map, options_map = parse_mapping(StringIO(mapping)) else: method_map, options_map = [], {} for pattern, method, options in mapping: method_map.append((pattern, method)) options_map[pattern] = options or {} mappings.append((path, method_map, options_map)) else: for path in self.input_paths: mappings.append((path, DEFAULT_MAPPING, {})) return mappings def check_message_extractors(dist, name, value): """Validate the ``message_extractors`` keyword argument to ``setup()``. :param dist: the distutils/setuptools ``Distribution`` object :param name: the name of the keyword argument (should always be "message_extractors") :param value: the value of the keyword argument :raise `DistutilsSetupError`: if the value is not valid """ assert name == 'message_extractors' if not isinstance(value, dict): raise DistutilsSetupError('the value of the "message_extractors" ' 'parameter must be a dictionary') class init_catalog(Command): """New catalog initialization command for use in ``setup.py`` scripts. If correctly installed, this command is available to Setuptools-using setup scripts automatically. For projects using plain old ``distutils``, the command needs to be registered explicitly in ``setup.py``:: from babel.messages.frontend import init_catalog setup( ... cmdclass = {'init_catalog': init_catalog} ) """ description = 'create a new catalog based on a POT file' user_options = [ ('domain=', 'D', "domain of PO file (default 'messages')"), ('input-file=', 'i', 'name of the input file'), ('output-dir=', 'd', 'path to output directory'), ('output-file=', 'o', "name of the output file (default " "'<output_dir>/<locale>/LC_MESSAGES/<domain>.po')"), ('locale=', 'l', 'locale for the new localized catalog'), ('width=', 'w', 'set output line width (default 76)'), ('no-wrap', None, 'do not break long message lines, longer than the output line width, ' 'into several lines'), ] boolean_options = ['no-wrap'] def initialize_options(self): self.output_dir = None self.output_file = None self.input_file = None self.locale = None self.domain = 'messages' self.no_wrap = False self.width = None def finalize_options(self): if not self.input_file: raise DistutilsOptionError('you must specify the input file') if not self.locale: raise DistutilsOptionError('you must provide a locale for the ' 'new catalog') try: self._locale = Locale.parse(self.locale) except UnknownLocaleError as e: raise DistutilsOptionError(e) if not self.output_file and not self.output_dir: raise DistutilsOptionError('you must specify the output directory') if not self.output_file: self.output_file = os.path.join(self.output_dir, self.locale, 'LC_MESSAGES', self.domain + '.po') if not os.path.exists(os.path.dirname(self.output_file)): os.makedirs(os.path.dirname(self.output_file)) if self.no_wrap and self.width: raise DistutilsOptionError("'--no-wrap' and '--width' are mutually " "exclusive") if not self.no_wrap and not self.width: self.width = 76 elif self.width is not None: self.width = int(self.width) def run(self): self.log.info( 'creating catalog %s based on %s', self.output_file, self.input_file ) with open(self.input_file, 'rb') as infile: # Although reading from the catalog template, read_po must be fed # the locale in order to correctly calculate plurals catalog = read_po(infile, locale=self.locale) catalog.locale = self._locale catalog.revision_date = datetime.now(LOCALTZ) catalog.fuzzy = False with open(self.output_file, 'wb') as outfile: write_po(outfile, catalog, width=self.width) class update_catalog(Command): """Catalog merging command for use in ``setup.py`` scripts. If correctly installed, this command is available to Setuptools-using setup scripts automatically. For projects using plain old ``distutils``, the command needs to be registered explicitly in ``setup.py``:: from babel.messages.frontend import update_catalog setup( ... cmdclass = {'update_catalog': update_catalog} ) .. versionadded:: 0.9 """ description = 'update message catalogs from a POT file' user_options = [ ('domain=', 'D', "domain of PO file (default 'messages')"), ('input-file=', 'i', 'name of the input file'), ('output-dir=', 'd', 'path to base directory containing the catalogs'), ('output-file=', 'o', "name of the output file (default " "'<output_dir>/<locale>/LC_MESSAGES/<domain>.po')"), ('omit-header', None, "do not include msgid "" entry in header"), ('locale=', 'l', 'locale of the catalog to compile'), ('width=', 'w', 'set output line width (default 76)'), ('no-wrap', None, 'do not break long message lines, longer than the output line width, ' 'into several lines'), ('ignore-obsolete=', None, 'whether to omit obsolete messages from the output'), ('no-fuzzy-matching', 'N', 'do not use fuzzy matching'), ('update-header-comment', None, 'update target header comment'), ('previous', None, 'keep previous msgids of translated messages'), ] boolean_options = [ 'omit-header', 'no-wrap', 'ignore-obsolete', 'no-fuzzy-matching', 'previous', 'update-header-comment', ] def initialize_options(self): self.domain = 'messages' self.input_file = None self.output_dir = None self.output_file = None self.omit_header = False self.locale = None self.width = None self.no_wrap = False self.ignore_obsolete = False self.no_fuzzy_matching = False self.update_header_comment = False self.previous = False def finalize_options(self): if not self.input_file: raise DistutilsOptionError('you must specify the input file') if not self.output_file and not self.output_dir: raise DistutilsOptionError('you must specify the output file or ' 'directory') if self.output_file and not self.locale: raise DistutilsOptionError('you must specify the locale') if self.no_wrap and self.width: raise DistutilsOptionError("'--no-wrap' and '--width' are mutually " "exclusive") if not self.no_wrap and not self.width: self.width = 76 elif self.width is not None: self.width = int(self.width) if self.no_fuzzy_matching and self.previous: self.previous = False def run(self): po_files = [] if not self.output_file: if self.locale: po_files.append((self.locale, os.path.join(self.output_dir, self.locale, 'LC_MESSAGES', self.domain + '.po'))) else: for locale in os.listdir(self.output_dir): po_file = os.path.join(self.output_dir, locale, 'LC_MESSAGES', self.domain + '.po') if os.path.exists(po_file): po_files.append((locale, po_file)) else: po_files.append((self.locale, self.output_file)) if not po_files: raise DistutilsOptionError('no message catalogs found') domain = self.domain if not domain: domain = os.path.splitext(os.path.basename(self.input_file))[0] with open(self.input_file, 'rb') as infile: template = read_po(infile) for locale, filename in po_files: self.log.info('updating catalog %s based on %s', filename, self.input_file) with open(filename, 'rb') as infile: catalog = read_po(infile, locale=locale, domain=domain) catalog.update( template, self.no_fuzzy_matching, update_header_comment=self.update_header_comment ) tmpname = os.path.join(os.path.dirname(filename), tempfile.gettempprefix() + os.path.basename(filename)) try: with open(tmpname, 'wb') as tmpfile: write_po(tmpfile, catalog, omit_header=self.omit_header, ignore_obsolete=self.ignore_obsolete, include_previous=self.previous, width=self.width) except: os.remove(tmpname) raise try: os.rename(tmpname, filename) except OSError: # We're probably on Windows, which doesn't support atomic # renames, at least not through Python # If the error is in fact due to a permissions problem, that # same error is going to be raised from one of the following # operations os.remove(filename) shutil.copy(tmpname, filename) os.remove(tmpname) class CommandLineInterface(object): """Command-line interface. This class provides a simple command-line interface to the message extraction and PO file generation functionality. """ usage = '%%prog %s [options] %s' version = '%%prog %s' % VERSION commands = { 'compile': 'compile message catalogs to MO files', 'extract': 'extract messages from source files and generate a POT file', 'init': 'create new message catalogs from a POT file', 'update': 'update existing message catalogs from a POT file' } command_classes = { 'compile': compile_catalog, 'extract': extract_messages, 'init': init_catalog, 'update': update_catalog, } log = None # Replaced on instance level def run(self, argv=None): """Main entry point of the command-line interface. :param argv: list of arguments passed on the command-line """ if argv is None: argv = sys.argv self.parser = optparse.OptionParser(usage=self.usage % ('command', '[args]'), version=self.version) self.parser.disable_interspersed_args() self.parser.print_help = self._help self.parser.add_option('--list-locales', dest='list_locales', action='store_true', help="print all known locales and exit") self.parser.add_option('-v', '--verbose', action='store_const', dest='loglevel', const=logging.DEBUG, help='print as much as possible') self.parser.add_option('-q', '--quiet', action='store_const', dest='loglevel', const=logging.ERROR, help='print as little as possible') self.parser.set_defaults(list_locales=False, loglevel=logging.INFO) options, args = self.parser.parse_args(argv[1:]) self._configure_logging(options.loglevel) if options.list_locales: identifiers = localedata.locale_identifiers() longest = max([len(identifier) for identifier in identifiers]) identifiers.sort() format = u'%%-%ds %%s' % (longest + 1) for identifier in identifiers: locale = Locale.parse(identifier) output = format % (identifier, locale.english_name) print(output.encode(sys.stdout.encoding or getpreferredencoding() or 'ascii', 'replace')) return 0 if not args: self.parser.error('no valid command or option passed. ' 'Try the -h/--help option for more information.') cmdname = args[0] if cmdname not in self.commands: self.parser.error('unknown command "%s"' % cmdname) cmdinst = self._configure_command(cmdname, args[1:]) return cmdinst.run() def _configure_logging(self, loglevel): self.log = logging.getLogger('babel') self.log.setLevel(loglevel) # Don't add a new handler for every instance initialization (#227), this # would cause duplicated output when the CommandLineInterface as an # normal Python class. if self.log.handlers: handler = self.log.handlers[0] else: handler = logging.StreamHandler() self.log.addHandler(handler) handler.setLevel(loglevel) formatter = logging.Formatter('%(message)s') handler.setFormatter(formatter) def _help(self): print(self.parser.format_help()) print("commands:") longest = max([len(command) for command in self.commands]) format = " %%-%ds %%s" % max(8, longest + 1) commands = sorted(self.commands.items()) for name, description in commands: print(format % (name, description)) def _configure_command(self, cmdname, argv): """ :type cmdname: str :type argv: list[str] """ cmdclass = self.command_classes[cmdname] cmdinst = cmdclass() if self.log: cmdinst.log = self.log # Use our logger, not distutils'. assert isinstance(cmdinst, Command) cmdinst.initialize_options() parser = optparse.OptionParser( usage=self.usage % (cmdname, ''), description=self.commands[cmdname] ) as_args = getattr(cmdclass, "as_args", ()) for long, short, help in cmdclass.user_options: name = long.strip("=") default = getattr(cmdinst, name.replace('-', '_')) strs = ["--%s" % name] if short: strs.append("-%s" % short) strs.extend(cmdclass.option_aliases.get(name, ())) choices = cmdclass.option_choices.get(name, None) if name == as_args: parser.usage += "<%s>" % name elif name in cmdclass.boolean_options: parser.add_option(*strs, action="store_true", help=help) elif name in cmdclass.multiple_value_options: parser.add_option(*strs, action="append", help=help, choices=choices) else: parser.add_option(*strs, help=help, default=default, choices=choices) options, args = parser.parse_args(argv) if as_args: setattr(options, as_args.replace('-', '_'), args) for key, value in vars(options).items(): setattr(cmdinst, key, value) try: cmdinst.ensure_finalized() except DistutilsOptionError as err: parser.error(str(err)) return cmdinst def main(): return CommandLineInterface().run(sys.argv) def parse_mapping(fileobj, filename=None): """Parse an extraction method mapping from a file-like object. >>> buf = StringIO(''' ... [extractors] ... custom = mypackage.module:myfunc ... ... # Python source files ... [python: **.py] ... ... # Genshi templates ... [genshi: **/templates/**.html] ... include_attrs = ... [genshi: **/templates/**.txt] ... template_class = genshi.template:TextTemplate ... encoding = latin-1 ... ... # Some custom extractor ... [custom: **/custom/*.*] ... ''') >>> method_map, options_map = parse_mapping(buf) >>> len(method_map) 4 >>> method_map[0] ('**.py', 'python') >>> options_map['**.py'] {} >>> method_map[1] ('**/templates/**.html', 'genshi') >>> options_map['**/templates/**.html']['include_attrs'] '' >>> method_map[2] ('**/templates/**.txt', 'genshi') >>> options_map['**/templates/**.txt']['template_class'] 'genshi.template:TextTemplate' >>> options_map['**/templates/**.txt']['encoding'] 'latin-1' >>> method_map[3] ('**/custom/*.*', 'mypackage.module:myfunc') >>> options_map['**/custom/*.*'] {} :param fileobj: a readable file-like object containing the configuration text to parse :see: `extract_from_directory` """ extractors = {} method_map = [] options_map = {} parser = RawConfigParser() parser._sections = OrderedDict(parser._sections) # We need ordered sections if PY2: parser.readfp(fileobj, filename) else: parser.read_file(fileobj, filename) for section in parser.sections(): if section == 'extractors': extractors = dict(parser.items(section)) else: method, pattern = [part.strip() for part in section.split(':', 1)] method_map.append((pattern, method)) options_map[pattern] = dict(parser.items(section)) if extractors: for idx, (pattern, method) in enumerate(method_map): if method in extractors: method = extractors[method] method_map[idx] = (pattern, method) return method_map, options_map def parse_keywords(strings=[]): """Parse keywords specifications from the given list of strings. >>> kw = sorted(parse_keywords(['_', 'dgettext:2', 'dngettext:2,3', 'pgettext:1c,2']).items()) >>> for keyword, indices in kw: ... print((keyword, indices)) ('_', None) ('dgettext', (2,)) ('dngettext', (2, 3)) ('pgettext', ((1, 'c'), 2)) """ keywords = {} for string in strings: if ':' in string: funcname, indices = string.split(':') else: funcname, indices = string, None if funcname not in keywords: if indices: inds = [] for x in indices.split(','): if x[-1] == 'c': inds.append((int(x[:-1]), 'c')) else: inds.append(int(x)) indices = tuple(inds) keywords[funcname] = indices return keywords if __name__ == '__main__': main()
Cancel