1. Daniel Greenfeld
  2. transifex

Commits

Diego Búrigo Zacarão  committed 2b3c34d

Added jsonmap addon

- This addon is used to create and store the file mapping
used to convert/migrate components into resources from 0.9.x
to 1.0 versions.
- The mapping creation can be done using the 'txcreatemap'
management command in the 0.9.1 or greater versions.
- Include south migration

  • Participants
  • Parent commits cb40083
  • Branches default

Comments (0)

Files changed (9)

File transifex/addons/jsonmap/__init__.py

View file
  • Ignore whitespace
+# -*- coding: utf-8 -*-
+
+class Meta:
+    author = "Indifex"
+    title = "Transifex jsonmap"
+    description = ("Addon used to create and store the file mapping used to "
+        "convert/migrate components into resources from 0.9.x to the 1.0 "
+        "version of Transifex.")
+

File transifex/addons/jsonmap/management/__init__.py

  • Ignore whitespace
Empty file added.

File transifex/addons/jsonmap/management/commands/__init__.py

  • Ignore whitespace
Empty file added.

File transifex/addons/jsonmap/management/commands/txcreatemap.py

View file
  • Ignore whitespace
+import os
+from optparse import make_option
+
+from django.core.management.base import CommandError, BaseCommand
+from django.db.models import Q, get_model
+from django.template.defaultfilters import slugify
+from django.utils import simplejson
+
+from txcommon import version_full as VERSION
+from txcommon.log import logger
+
+from jsonmap.models import JSONMap
+
+Project = get_model('projects', 'Project')
+
+def get_source_file_for_file(filename, source_files):
+    """
+    Find the related source file (POT) for a file (PO); useful when it has 
+    multiple source files.
+
+    This method gets a filename and the related source_files as parameters 
+    and tries to discover the related POT file using two methods:
+
+    1. Trying to find a POT file with the same base path that the PO.
+        Example: /foo/bar.pot and /foo/baz.po match on this method.
+
+    2. Trying to find a POT file with the same domain that the PO in any
+        directory.
+
+        Example: /foo/bar.pot and /foo/baz/bar.po match on this method.
+        The domain in this case is 'bar'.
+
+    If no POT is found the method returns None.
+
+    """
+    # For filename='/foo/bar.po'
+    fb = os.path.basename(filename) # 'bar.po'
+    fp = filename.split(fb)[0]        # '/foo/'
+
+    # Find the POT with the same domain or path that the filename,
+    # if the component has more that one POT file
+    if len(source_files) > 1:
+        for source in source_files:
+            sb = os.path.basename(source)
+            if sb.endswith('.pot'):
+                sb = sb[:-1]
+            pb = source.split(sb)[0]
+            if pb==fp or sb==fb:
+                return source
+    elif len(source_files) == 1:
+        return source_files[0]
+
+class Command(BaseCommand):
+
+    option_list = BaseCommand.option_list + (
+        make_option('--anyversion', '-a', action="store_true", default=False, 
+            dest='anyversion', help='Skip Transifex version check.'),
+    )
+
+    args = '<project_slug project_slug ...>'
+    help = """
+    Create a JSON formatted mapping of the POT files and theirs translations
+    (PO) for projects, converting components into resources. Each project
+    component has it own mapping which is stored in the database within the
+    ``jsonmap.models.JSONMap`` model.
+
+    This mapping can be used by the CLI app of Transifex as the ``.tx/txdata``
+    file content in the remote repository.
+
+    Project slugs can be passed by argument to map specific projects. If no
+    argument is passed the mapping will happen for all the projects in the
+    database.
+
+    This mapping is used to migrate versions <= 0.9.x to the 1.0 version of
+    Transifex.
+    """
+
+    def handle(self, *args, **options):
+        anyversion = options.get('anyversion')
+
+        if not anyversion:
+            if int(VERSION.split('.')[0]) >= 1:
+                raise CommandError("This command can't be used in Transifex "
+                    "versions greater than 0.9.x. Your current version is "
+                    "'%s'. For skipping this check use the option '-a'."
+                    % VERSION)
+
+        if len(args) == 0:
+            projects = Project.objects.all()
+        else:
+            projects = Project.objects.filter(slug__in=args)
+            if not projects:
+                raise CommandError("No project found with the given "
+                    "slug(s): %s" % ', '.join(args))
+
+        nprojects = 0
+        without_resources = []
+        for p in projects:
+            repo_resources = []
+            for c in p.component_set.all():
+                # Initialize some things
+                resources = []
+                translations = {}
+                c.unit._init_browser()
+
+                # Get source files; it might be a .pot or a .po with the same 
+                # language code as the one set on component source language.
+                source_files = c.pofiles.filter(enabled=True).filter(
+                    Q(is_pot=True) | Q(language_code=c.source_lang))
+
+                logger.debug("Mapping %s" % c.full_name)
+
+                for k, source_file in enumerate(source_files):
+                    r_slug = "%s-%s" % (c.slug, 
+                        source_file.filename.replace('/','-').replace('.','-'))
+                    if len(r_slug) > 30:
+                        r_slug = r_slug[-30:] + '_%s' % str(k)
+
+                    logger.debug("Resource: %s" % r_slug)
+
+                    # Map each source file as a resource
+                    resources.append({
+                        'source_file': source_file.filename,
+                        'source_lang': "en", 
+                        'resource_slug': r_slug,
+                        '_repo_path': c.unit.browser.path,
+                    })
+
+                    # Temp var to associate translation files with the 
+                    # corresponding source file; useful for components with
+                    # multiple source files.
+                    translations.update({source_file.filename:{}})
+
+                # List of source file names; used on get_source_file_for_file()
+                source_filenames = [f.filename for f in source_files]
+
+                # Going through all the translation files of the related 
+                # component, which have a language associated with.
+                for po in c.pofiles.filter(enabled=True).filter(
+                    language__isnull=False).exclude(language_code=c.source_lang):
+
+                    # Get related source file for the given translation file
+                    rsf = get_source_file_for_file(po.filename, source_filenames)
+                    if rsf:
+                        # Add translation file to the temp mapping
+                        translations[rsf].update(
+                            {po.language_code:{'file':po.filename}})
+
+                # Finally add the translations to the related resource
+                for r in resources:
+                    r['translations']=translations.get(r['source_file'], {})
+
+                # Only save the JSON file if there is at least one resource for 
+                # the project.
+                if resources:
+                    data = { 'meta': {'project_slug': p.slug},
+                             'resources':resources}
+                    j = JSONMap.objects.get_or_create(project=p, slug=c.slug)[0]
+                    j.content = simplejson.dumps(data, indent=2)
+                    j.save()
+                    repo_resources.append(resources)
+
+            if repo_resources:
+                nprojects+=1
+            else:
+                without_resources.append(p.slug)
+
+        print "Projects with at least one resource created: %s" % nprojects
+        print "Projects with no resources: %s" % len(without_resources)
+
+

File transifex/addons/jsonmap/management/commands/txmigratemap.py

View file
  • Ignore whitespace
+import os
+from optparse import make_option
+
+from django.conf import settings
+from django.core.management.base import CommandError, BaseCommand
+from django.db import models
+
+from txcommon.commands import run_command
+from txcommon.log import logger
+
+from jsonmap.models import JSONMap
+
+
+class Command(BaseCommand):
+
+    option_list = BaseCommand.option_list + (
+        make_option('--datadir', '-d', default='.tx', dest='datadir', 
+            help="Directoty for storing tx data. Default '.tx'."),
+        make_option('--filename', '-f', default='txdata', dest='filename', 
+            help="Filename target for JSON mapping. Default 'txdata'.")
+    )
+
+    args = '<project_slug project_slug ...>'
+    help = "Create resources based on a JSON formatted mapping."
+
+    def handle(self, *args, **options):
+
+        # OMG!1! Dirty fix for circular importing issues. Didn't want to dig
+        # into it because it's probably not worth, once it's a tmp code.
+        from resources.formats import get_i18n_type_from_file
+        from resources.formats.pofile import POHandler
+        from languages.models import Language
+        from projects.models import Project
+        from resources.models import Resource
+
+        datadir = options.get('datadir')
+        filename = options.get('filename')
+
+        msg = None
+        if len(args) == 0:
+            jsonmaps = JSONMap.objects.all()
+        else:
+            jsonmaps = JSONMap.objects.filter(project__slug__in=args)
+            if not jsonmaps:
+                msg = "No mapping found for given project slug(s): %s" % ', '.join(args)
+
+        if not jsonmaps:
+            raise CommandError(msg or "No mapping found in the database.")
+
+        for jsonmap in jsonmaps:
+            for r in jsonmap.loads(True)['resources']:
+                logger.debug("Pushing resource: %s" % r.get('resource_slug'))
+
+                project = jsonmap.project
+
+                # Path for cached files of project.component
+                path = os.path.join(settings.MSGMERGE_DIR,
+                    '%s.%s' % (project.slug, jsonmap.slug))
+
+                if os.path.exists(path):
+
+                    resource_slug = r['resource_slug']
+                    language = Language.objects.by_code_or_alias_or_none(
+                        r['source_lang'])
+
+                    # Create resource and load source language
+                    if language:
+                        resource, created = Resource.objects.get_or_create(
+                                slug = resource_slug,
+                                source_language = language,
+                                project = project,
+                                name = resource_slug)
+
+                        source_file = os.path.join(path, r['source_file'])
+                        resource.i18n_type = get_i18n_type_from_file(source_file)
+                        resource.save()
+
+                        logger.debug("Inserting source strings from %s (%s) to "
+                            "'%s' (%s)." % (r['source_file'], language.code,
+                            resource.slug, project))
+
+                        fhandler = POHandler(filename=source_file)
+                        fhandler.bind_resource(resource)
+                        fhandler.set_language(language)
+
+                        try:
+                            fhandler.contents_check(fhandler.filename)
+                            fhandler.parse_file(True)
+                            strings_added, strings_updated = fhandler.save2db(True)
+                        except Exception, e:
+                            print "Could not import file '%s': %s" % \
+                                (source_file, str(e))
+
+                        logger.debug("Inserting translations for '%s' (%s)." 
+                            % (resource.slug, project))
+
+                        # Load translations
+                        for code, f in r['translations'].items():
+                            language = Language.objects.by_code_or_alias_or_none(code)
+                            if language:
+                                translation_file = os.path.join(path, f['file'])
+                                fhandler = POHandler(filename=translation_file)
+                                fhandler.set_language(language)
+                                fhandler.bind_resource(resource)
+                                fhandler.contents_check(fhandler.filename)
+
+                                try:
+                                    fhandler.parse_file()
+                                    strings_added, strings_updated = fhandler.save2db()
+                                except Exception, e:
+                                    print "Could not import file '%s': %s" % \
+                                        (translation_file, str(e))
+                else:
+                    logger.debug("Mapping '%s' does have canched files at "
+                        "%s." % path)
+
+

File transifex/addons/jsonmap/migrations/0001_initial.py

View file
  • Ignore whitespace
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'JSONMap'
+        db.create_table('jsonmap_jsonmap', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('slug', self.gf('django.db.models.fields.TextField')(max_length=50)),
+            ('content', self.gf('txcommon.db.models.CompressedTextField')(null=False, blank=False)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+            ('project', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['projects.Project'])),
+        ))
+        db.send_create_signal('jsonmap', ['JSONMap'])
+
+        # Adding unique constraint on 'JSONMap', fields ['project', 'slug']
+        db.create_unique('jsonmap_jsonmap', ['project_id', 'slug'])
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'JSONMap'
+        db.delete_table('jsonmap_jsonmap')
+
+        # Removing unique constraint on 'JSONMap', fields ['project', 'slug']
+        db.delete_unique('jsonmap_jsonmap', ['project_id', 'slug'])
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'jsonmap.jsonmap': {
+            'Meta': {'unique_together': "(('project', 'slug'),)", 'object_name': 'JSONMap'},
+            'content': ('txcommon.db.models.CompressedTextField', [], {'null': 'False', 'blank': 'False'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['projects.Project']"}),
+            'slug': ('django.db.models.fields.TextField', [], {'max_length': '50'})
+        },
+        'projects.project': {
+            'Meta': {'object_name': 'Project'},
+            'anyone_submit': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'bug_tracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}),
+            'feed': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'long_description': ('django.db.models.fields.TextField', [], {'max_length': '1000', 'blank': 'True'}),
+            'long_description_html': ('django.db.models.fields.TextField', [], {'max_length': '1000', 'blank': 'True'}),
+            'maintainers': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'projects_maintaining'", 'null': 'True', 'to': "orm['auth.User']"}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'outsource': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['projects.Project']", 'null': 'True', 'blank': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}),
+            'private': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '30', 'db_index': 'True'}),
+            'tags': ('tagging.fields.TagField', [], {})
+        }
+    }
+
+    complete_apps = ['jsonmap']

File transifex/addons/jsonmap/migrations/__init__.py

  • Ignore whitespace
Empty file added.

File transifex/addons/jsonmap/models.py

View file
  • Ignore whitespace
+# -*- coding: utf-8 -*-
+import copy
+import re
+from django.conf import settings
+from django.db import models
+from django.utils import simplejson
+from django.utils.translation import ugettext_lazy as _
+
+from txcommon.db.models import CompressedTextField
+from txcommon.log import logger
+
+from jsonmap.utils import remove_attrs_startwith
+
+
+class JSONMap(models.Model):
+    """
+    Store the JSON mapping used to among resources and its translation files
+    """
+    slug = models.TextField(null=False, blank=False, max_length=50,
+        help_text="Slug for the mapping. Usually the same as the old "
+        "component slug.")
+    content = CompressedTextField(null=False, blank=False,
+        help_text="Mapping in JSON format.")
+    created = models.DateTimeField(auto_now_add=True, editable=False)
+    modified = models.DateTimeField(auto_now=True, editable=False)
+
+    # ForeignKeys
+    project = models.ForeignKey('projects.Project', null=False, blank=False,
+        help_text="Project which the JSON mapping belongs to.")
+
+    def __unicode__(self):
+        return '%s.%s' % (self.project.slug, self.slug)
+
+    def __repr__(self):
+        return '<JSONMap: %s.%s>' % (self.project.slug, self.slug)
+
+    class Meta:
+        unique_together = ("project", "slug")
+        verbose_name = 'JSONMap'
+        verbose_name_plural = 'JSONMaps'
+        ordering  = ('project__name',)
+        get_latest_by = 'created'
+
+
+    def dumps(self, with_tmp_attr=False):
+       """
+       Return the JSON formatted ``self.content`` with no tmp attributes if 
+       with_tmp_attr is False.
+       """
+       return simplejson.dumps(self.loads(with_tmp_attr), indent=2,
+                encoding=settings.DEFAULT_CHARSET)
+
+
+    def dumps_to_file(self, filename, with_tmp_attr=False):
+        """
+        Write ``self.dumps()`` result into a JSON file to the file system.
+        """
+        tfile = None
+        try:
+            tfile = open(filename, 'w')
+            tfile.write(self.dumps(with_tmp_attr))
+        except:
+            pass
+        if tfile:
+            tfile.close()
+
+
+    def loads(self, with_tmp_attr=False):
+        """
+        Deserialize ``self.content`` to a Python dictionary.
+
+        Remove temporary attribute if with_tmp_attr is False. Attributes that
+        start with '_', such as '_repo_path', will be removed in such case.
+        """
+        try:
+            data = simplejson.loads(self.content,
+                encoding=settings.DEFAULT_CHARSET)
+        except ValueError:
+            data = eval(self.content)
+
+        if not with_tmp_attr:
+            remove_attrs_startwith(data, '_')
+
+        return data

File transifex/addons/jsonmap/utils.py

View file
  • Ignore whitespace
+def remove_attrs_startwith(dictionay, chars):
+    """
+    Remove attributes starting with ``chars`` from ``dictionary`` in place.
+    """
+    def for_list(val):
+        for v in val:
+            if isinstance(v, (list, tuple)):
+                for_list(v)
+            elif type(v) == dict:
+                remove_attrs_startwith(v, chars)
+
+    for key, val in dictionay.items():
+        if key.startswith(chars):
+            dictionay.pop(key)
+        elif type(val) != dict:
+            if isinstance(val, (list, tuple)):
+                for_list(val)
+        else:
+            remove_attrs_startwith(val, chars)