Ticket #6552: packagerChrome.py

File packagerChrome.py, 14.2 KB (added by BrentM, on 04/04/2018 at 07:49:55 PM)
Line 
1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5import errno
6import glob
7import io
8import json
9import os
10import re
11import struct
12import subprocess
13import sys
14import random
15import posixpath
16
17from packager import (readMetadata, getDefaultFileName, getBuildVersion,
18                      getTemplate, get_extension, Files, get_app_id)
19
20defaultLocale = 'en_US'
21
22
23def getIgnoredFiles(params):
24    return {'store.description'}
25
26
27def getPackageFiles(params):
28    result = {'_locales', 'icons', 'jquery-ui', 'lib', 'skin', 'ui', 'ext'}
29
30    if params['devenv']:
31        result.add('qunit')
32
33    baseDir = params['baseDir']
34
35    for file in os.listdir(baseDir):
36        if os.path.splitext(file)[1] in {'.json', '.js', '.html', '.xml'}:
37            result.add(file)
38    return result
39
40
41def processFile(path, data, params):
42    # We don't change anything yet, this function currently only exists here so
43    # that it can be overridden if necessary.
44    return data
45
46
47def makeIcons(files, filenames):
48    icons = {}
49    for filename in filenames:
50        try:
51            magic, width, height = struct.unpack_from('>8s8xii',
52                                                      files[filename])
53        except struct.error:
54            magic = None
55        if magic != '\x89PNG\r\n\x1a\n':
56            raise Exception(filename + ' is no valid PNG.')
57        if width != height:
58            print >>sys.stderr, 'Warning: %s size is %ix%i, icon should be square' % (filename, width, height)
59        icons[width] = filename
60    return icons
61
62
63def createScriptPage(params, template_name, script_option):
64    template = getTemplate(template_name, autoEscape=True)
65    return template.render(
66        basename=params['metadata'].get('general', 'basename'),
67        scripts=params['metadata'].get(*script_option).split()
68    ).encode('utf-8')
69
70
71def createManifest(params, files):
72    template = getTemplate('manifest.json.tmpl')
73    templateData = dict(params)
74
75    baseDir = templateData['baseDir']
76    metadata = templateData['metadata']
77
78    for opt in ('browserAction', 'pageAction'):
79        if not metadata.has_option('general', opt):
80            continue
81
82        icons = metadata.get('general', opt).split()
83        if not icons:
84            continue
85
86        if len(icons) == 1:
87            # ... = icon.png
88            icon, popup = icons[0], None
89        elif icons[-1].endswith('.html'):
90            if len(icons) == 2:
91                # ... = icon.png popup.html
92                icon, popup = icons
93            else:
94                # ... = icon-19.png icon-38.png popup.html
95                popup = icons.pop()
96                icon = makeIcons(files, icons)
97        else:
98            # ... = icon-16.png icon-32.png icon-48.png
99            icon = makeIcons(files, icons)
100            popup = None
101
102        templateData[opt] = {'icon': icon, 'popup': popup}
103
104    if metadata.has_option('general', 'icons'):
105        templateData['icons'] = makeIcons(files,
106                                          metadata.get('general', 'icons').split())
107
108    if metadata.has_option('general', 'permissions'):
109        templateData['permissions'] = metadata.get('general', 'permissions').split()
110
111    if metadata.has_option('general', 'optionalPermissions'):
112        templateData['optionalPermissions'] = metadata.get(
113            'general', 'optionalPermissions').split()
114
115    if metadata.has_option('general', 'backgroundScripts'):
116        templateData['backgroundScripts'] = metadata.get(
117            'general', 'backgroundScripts').split()
118        if params['devenv']:
119            templateData['backgroundScripts'].append('devenvPoller__.js')
120
121    if metadata.has_option('general', 'webAccessible') and metadata.get('general', 'webAccessible') != '':
122        templateData['webAccessible'] = metadata.get('general',
123                                                     'webAccessible').split()
124
125    if metadata.has_option('general', 'externallyConnectable') and metadata.get('general', 'externallyConnectable') != '':
126        templateData['externallyConnectable'] = metadata.get('general', 'externallyConnectable').split()
127
128    if metadata.has_section('contentScripts'):
129        contentScripts = []
130        for run_at, scripts in metadata.items('contentScripts'):
131            if scripts == '':
132                continue
133            contentScripts.append({
134                'matches': ['http://*/*', 'https://*/*'],
135                'js': scripts.split(),
136                'run_at': run_at,
137                'all_frames': True,
138                'match_about_blank': True,
139            })
140        templateData['contentScripts'] = contentScripts
141    if params['type'] == 'gecko':
142        templateData['app_id'] = get_app_id(params['releaseBuild'], metadata)
143
144    manifest = template.render(templateData)
145
146    # Normalize JSON structure
147    licenseComment = re.compile(r'/\*.*?\*/', re.S)
148    data = json.loads(re.sub(licenseComment, '', manifest, 1))
149    if '_dummy' in data:
150        del data['_dummy']
151    manifest = json.dumps(data, sort_keys=True, indent=2)
152
153    return manifest.encode('utf-8')
154
155
156def toJson(data):
157    return json.dumps(
158        data, ensure_ascii=False, sort_keys=True,
159        indent=2, separators=(',', ': ')
160    ).encode('utf-8') + '\n'
161
162
163def create_bundles(params, files, bundle_tests):
164    base_extension_path = params['baseDir']
165    info_templates = {
166        'chrome': 'chromeInfo.js.tmpl',
167        'edge': 'edgeInfo.js.tmpl',
168        'gecko': 'geckoInfo.js.tmpl'
169    }
170
171    # Historically we didn't use relative paths when requiring modules, so in
172    # order for webpack to know where to find them we need to pass in a list of
173    # resolve paths. Going forward we should always use relative paths, once we
174    # do that consistently this can be removed. See issues 5760, 5761 and 5762.
175    resolve_paths = [os.path.join(base_extension_path, dir, 'lib')
176                     for dir in ['', 'adblockpluscore', 'adblockplusui']]
177
178    info_template = getTemplate(info_templates[params['type']])
179    info_module = info_template.render(
180        basename=params['metadata'].get('general', 'basename'),
181        version=params['metadata'].get('general', 'version')
182    ).encode('utf-8')
183
184    configuration = {
185        'bundles': [],
186        'extension_path': base_extension_path,
187        'info_module': info_module,
188        'resolve_paths': resolve_paths,
189    }
190
191    for item in params['metadata'].items('bundles'):
192        name, value = item
193        base_item_path = os.path.dirname(item.source)
194
195        bundle_file = os.path.relpath(os.path.join(base_item_path, name),
196                                      base_extension_path)
197        entry_files = [os.path.join(base_item_path, module_path)
198                       for module_path in value.split()]
199        configuration['bundles'].append({
200            'bundle_name': bundle_file,
201            'entry_points': entry_files,
202        })
203
204    if bundle_tests:
205        qunit_path = os.path.join(base_extension_path, 'qunit')
206        qunit_files = ([os.path.join(qunit_path, 'common.js')] +
207                       glob.glob(os.path.join(qunit_path, 'tests', '*.js')))
208        configuration['bundles'].append({
209            'bundle_name': 'qunit/tests.js',
210            'entry_points': qunit_files
211        })
212
213    cmd = ['node', os.path.join(os.path.dirname(__file__), 'webpack_runner.js')]
214    process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
215                               stdin=subprocess.PIPE)
216    output = process.communicate(input=toJson(configuration))[0]
217    if process.returncode != 0:
218        raise subprocess.CalledProcessError(process.returncode, cmd=cmd)
219    output = json.loads(output)
220
221    # Clear the mapping for any files included in a bundle, to avoid them being
222    # duplicated in the build.
223    for to_ignore in output['included']:
224        files.pop(to_ignore, None)
225
226    for bundle in output['files']:
227        files[bundle] = output['files'][bundle].encode('utf-8')
228
229
230def import_locales(params, files):
231    for item in params['metadata'].items('import_locales'):
232        filename = item[0]
233        for sourceFile in glob.glob(os.path.join(os.path.dirname(item.source),
234                                                 *filename.split('/'))):
235            keys = item[1]
236            locale = sourceFile.split(os.path.sep)[-2]
237            targetFile = posixpath.join('_locales', locale, 'messages.json')
238            data = json.loads(files.get(targetFile, '{}').decode('utf-8'))
239
240            try:
241                with io.open(sourceFile, 'r', encoding='utf-8') as handle:
242                    sourceData = json.load(handle)
243
244                # Resolve wildcard imports
245                if keys == '*':
246                    importList = sourceData.keys()
247                    importList = filter(lambda k: not k.startswith('_'), importList)
248                    keys = ' '.join(importList)
249
250                for stringID in keys.split():
251                    if stringID in sourceData:
252                        if stringID in data:
253                            print ('Warning: locale string {} defined multiple'
254                                   ' times').format(stringID)
255
256                        data[stringID] = sourceData[stringID]
257            except Exception as e:
258                print 'Warning: error importing locale data from %s: %s' % (sourceFile, e)
259
260            files[targetFile] = toJson(data)
261
262
263def truncate(text, length_limit):
264    if len(text) <= length_limit:
265        return text
266    return text[:length_limit - 1].rstrip() + u'\u2026'
267
268
269def fix_translations_for_chrome(files):
270    defaults = {}
271    data = json.loads(files['_locales/%s/messages.json' % defaultLocale])
272    for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']):
273        name = match.group(1)
274        defaults[name] = data[name]
275
276    limits = {}
277    manifest = json.loads(files['manifest.json'])
278    for key, limit in (('name', 45), ('description', 132), ('short_name', 12)):
279        match = re.search(r'__MSG_(\S+)__', manifest.get(key, ''))
280        if match:
281            limits[match.group(1)] = limit
282
283    for path in list(files):
284        match = re.search(r'^_locales/(?:es_(AR|CL|(MX))|[^/]+)/(.*)', path)
285        if not match:
286            continue
287
288        # The Chrome Web Store requires messages used in manifest.json to
289        # be present in all languages, and enforces length limits on
290        # extension name and description.
291        is_latam, is_mexican, filename = match.groups()
292        if filename == 'messages.json':
293            data = json.loads(files[path])
294            for name, info in defaults.iteritems():
295                data.setdefault(name, info)
296            for name, limit in limits.iteritems():
297                info = data.get(name)
298                if info:
299                    info['message'] = truncate(info['message'], limit)
300            files[path] = toJson(data)
301
302        # Chrome combines Latin American dialects of Spanish into es-419.
303        if is_latam:
304            data = files.pop(path)
305            if is_mexican:
306                files['_locales/es_419/' + filename] = data
307
308
309def signBinary(zipdata, keyFile):
310    from Crypto.Hash import SHA
311    from Crypto.PublicKey import RSA
312    from Crypto.Signature import PKCS1_v1_5
313
314    try:
315        with open(keyFile, 'rb') as file:
316            key = RSA.importKey(file.read())
317    except IOError as e:
318        if e.errno != errno.ENOENT:
319            raise
320        key = RSA.generate(2048)
321        with open(keyFile, 'wb') as file:
322            file.write(key.exportKey('PEM'))
323
324    return PKCS1_v1_5.new(key).sign(SHA.new(zipdata))
325
326
327def getPublicKey(keyFile):
328    from Crypto.PublicKey import RSA
329    with open(keyFile, 'rb') as file:
330        return RSA.importKey(file.read()).publickey().exportKey('DER')
331
332
333def writePackage(outputFile, pubkey, signature, zipdata):
334    if isinstance(outputFile, basestring):
335        file = open(outputFile, 'wb')
336    else:
337        file = outputFile
338    if pubkey != None and signature != None:
339        file.write(struct.pack('<4sIII', 'Cr24', 2, len(pubkey), len(signature)))
340        file.write(pubkey)
341        file.write(signature)
342    file.write(zipdata)
343
344
345def add_devenv_requirements(files, metadata, params):
346    files.read(
347        os.path.join(os.path.dirname(__file__), 'chromeDevenvPoller__.js'),
348        relpath='devenvPoller__.js',
349    )
350    files['devenvVersion__'] = str(random.random())
351
352    if metadata.has_option('general', 'testScripts'):
353        files['qunit/index.html'] = createScriptPage(
354            params, 'testIndex.html.tmpl', ('general', 'testScripts')
355        )
356
357
358def createBuild(baseDir, type='chrome', outFile=None, buildNum=None, releaseBuild=False, keyFile=None, devenv=False):
359    metadata = readMetadata(baseDir, type)
360    version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum)
361
362    if outFile == None:
363        file_extension = get_extension(type, keyFile is not None)
364        outFile = getDefaultFileName(metadata, version, file_extension)
365
366    params = {
367        'type': type,
368        'baseDir': baseDir,
369        'releaseBuild': releaseBuild,
370        'version': version,
371        'devenv': devenv,
372        'metadata': metadata,
373    }
374
375    mapped = metadata.items('mapping') if metadata.has_section('mapping') else []
376    files = Files(getPackageFiles(params), getIgnoredFiles(params),
377                  process=lambda path, data: processFile(path, data, params))
378
379    files.readMappedFiles(mapped)
380    files.read(baseDir, skip=[opt for opt, _ in mapped])
381
382    if metadata.has_section('bundles'):
383        bundle_tests = devenv and metadata.has_option('general', 'testScripts')
384        create_bundles(params, files, bundle_tests)
385
386    if metadata.has_section('preprocess'):
387        files.preprocess(
388            [f for f, _ in metadata.items('preprocess')],
389            {'needsExt': True}
390        )
391
392    if metadata.has_section('import_locales'):
393        import_locales(params, files)
394
395    files['manifest.json'] = createManifest(params, files)
396    if type == 'chrome':
397        fix_translations_for_chrome(files)
398
399    if devenv:
400        add_devenv_requirements(files, metadata, params)
401
402    zipdata = files.zipToString()
403    signature = None
404    pubkey = None
405    if keyFile != None:
406        signature = signBinary(zipdata, keyFile)
407        pubkey = getPublicKey(keyFile)
408    writePackage(outFile, pubkey, signature, zipdata)