~launchpad-pqm/launchpad/devel : revision 13314.4.3

1

"""build.py - Minifies and creates the JS build directory."""

2

3

__metaclass__ = type

4

__all__ = []

5

6

7

import optparse

8

import os

9

import re

10

import shutil

11

import sys

12

13

from glob import glob

14

15

import cssutils

16

import pkg_resources

17

18

HERE = os.path.dirname(__file__)

19

BUILD_DIR = os.path.normpath(os.path.join(HERE, '..', '..', '..', 'build'))

20

DEFAULT_SRC_DIR = os.path.normpath(os.path.join(HERE, '..', '..', '..'))

21

PKG_SRC_DIR = pkg_resources.resource_filename(

22

pkg_resources.Requirement.parse("lazr-js"), "lazrjs")

23

24

ESCAPE_STAR_PROPERTY_RE = re.compile(r'\*([a-zA-Z0-9_-]+):')

25

UNESCAPE_STAR_PROPERTY_RE = re.compile(r'([a-zA-Z0-9_-]+)_ie_star_hack:')

26

URL_RE = re.compile("url\([ \"\']*([^ \"\']+)[ \"\']*\)")

27

28

from jsmin import JavascriptMinify

29

30

31

def relative_path(from_file, to_file):

32

"""Return the relative path between from_file and to_file."""

33

dir_from, base_from = os.path.split(from_file)

34

dir_to, base_to = os.path.split(to_file)

35

path = os.path.relpath(dir_to, dir_from)

36

if path == ".":

37

return base_to

38

return os.path.join(path, base_to)

39

40

41

class ComboFile:

42

"""A file made up of several combined files.

43

44

It offers support for detecting if the file needs updating and updating it

45

from it source files.

46

"""

47

48

def __init__(self, src_files, target_file):

49

self.src_files = src_files

50

self.target_file = target_file

51

52

def needs_update(self):

53

"""Return True when the file needs to be updated.

54

55

This is usually because the target file doesn't exist yet or because

56

one of the source file was modified.

57

"""

58

# If the target file doesn't exist, we need updating!

59

if not os.path.exists(self.target_file):

60

return True

61

62

# Check if the target file was modified after all the src files.

63

target_mtime = os.stat(self.target_file).st_mtime

64

for src_file in self.src_files:

65

if os.stat(src_file).st_mtime > target_mtime:

66

return True

67

else:

68

return False

69

70

def log(self, msg):

71

sys.stdout.write(msg + '\n')

72

73

def update(self):

74

"""Update the file from its source files."""

75

target_fh = open(self.target_file, 'w')

76

try:

77

for src_file in self.src_files:

78

self.log("Processing '%s'" % os.path.basename(src_file))

79

target_fh.write(self.get_file_header(src_file))

80

fh = open(src_file)

81

content = fh.read()

82

fh.close()

83

try:

84

target_fh.write(self.filter_file_content(content, src_file))

85

except Exception:

86

os.remove(self.target_file)

87

raise

88

finally:

89

target_fh.close()

90

91

def get_comment(self, msg):

92

"""Return a string wrapped in a comment to be include in the output.

93

94

Can be used to help annotate the output file.

95

"""

96

return ''

97

98

def get_file_header(self, path):

99

"""Return a string to include before outputting a file.

100

101

Can be used by subclasses to output a file reference in the combined

102

file. Default implementation returns nothing.

103

"""

104

return ''

105

106

def filter_file_content(self, file_content, path):

107

"""Hook to process the file content before being combined."""

108

return file_content

109

110

111

class JSComboFile(ComboFile):

112

"""ComboFile for JavaScript files.

113

114

Outputs the filename before each combined file and make sure that

115

each file content has a new line.

116

"""

117

118

def get_comment(self, msg):

119

return "// %s\n" % msg

120

121

def get_file_header(self, path):

122

return self.get_comment(relative_path(self.target_file, path))

123

124

def filter_file_content(self, file_content, path):

125

return file_content + '\n'

126

127

128

class CSSComboFile(ComboFile):

129

"""FileCombiner for CSS files.

130

131

It uses the cssutils.CSSParser to convert all url() instances

132

to the new location, and minify the result.

133

"""

134

135

def __init__(self, src_files, target_file, resource_prefix="",

136

minify=True, rewrite_urls=True):

137

super(CSSComboFile, self).__init__(src_files, target_file)

138

self.resource_prefix = resource_prefix.rstrip("/")

139

self.minify = minify

140

self.rewrite_urls = rewrite_urls

141

142

def get_comment(self, msg):

143

return "/* %s */\n" % msg

144

145

def get_file_header(self, path):

146

return self.get_comment(relative_path(self.target_file, path))

147

148

def filter_file_content(self, file_content, path):

149

"""URLs are made relative to the target and the CSS is minified."""

150

if self.rewrite_urls:

151

src_dir = os.path.dirname(path)

152

relative_parts = relative_path(self.target_file, src_dir).split(

153

os.path.sep)

154

def fix_relative_url(match):

155

url = match.group(1)

156

# Don't modify absolute URLs or 'data:' urls.

157

if (url.startswith("http") or

158

url.startswith("/") or

159

url.startswith("data:")):

160

return match.group(0)

161

parts = relative_parts + url.split("/")

162

result = []

163

for part in parts:

164

if part == ".." and result and result[-1] != "..":

165

result.pop(-1)

166

continue

167

result.append(part)

168

return "url(%s)" % "/".join(

169

filter(None, [self.resource_prefix] + result))

170

file_content = URL_RE.sub(fix_relative_url, file_content)

171

172

if self.minify:

173

old_serializer = cssutils.ser

174

cssutils.setSerializer(cssutils.serialize.CSSSerializer())

175

try:

176

cssutils.ser.prefs.useMinified()

177

178

stylesheet = ESCAPE_STAR_PROPERTY_RE.sub(

179

r'\1_ie_star_hack:', file_content)

180

parser = cssutils.CSSParser(raiseExceptions=True)

181

css = parser.parseString(stylesheet)

182

stylesheet = UNESCAPE_STAR_PROPERTY_RE.sub(

183

r'*\1:', css.cssText)

184

return stylesheet + "\n"

185

finally:

186

cssutils.setSerializer(old_serializer)

187

return file_content + "\n"

188

189

190

class Builder:

191

192

def __init__(self, name='lazr', build_dir=BUILD_DIR, src_dir=PKG_SRC_DIR,

193

extra_files=None, exclude_regex='', file_type='raw',

194

copy_yui_to=BUILD_DIR):

195

"""Create a new Builder.

196

197

:param name: The name of the package we are building. This will

198

be used to compute the standalone JS and CSS files.

199

:param build_dir: The directory containing the build tree.

200

:param src_dir: The directory containing the source files.

201

:param extra_files: List of files that should be bundled in the

202

standalone file.

203

:param exclude_regex: A regex that will exclude file paths from the

204

final rollup. -min and -debug versions will still be built.

205

:param file_type: A string specifying which type of files to include

206

in the final rollup. Default is to use the raw, unmodified JS

207

file. Possible values are 'raw', 'min', and 'debug'. File types

208

are identified by their basename suffix: foo.js, foo-min.js,

209

foo-debug.js, etc.

210

:param copy_yui_to: The absolute path (as a string) to the place where

211

the YUI modules should be copied to. Defaults to BUILD_DIR.

212

"""

213

self.name = name

214

self.build_dir = build_dir

215

self.src_dir = src_dir

216

# We need to support the case where this is being invoked directly

217

# from source rather than a package. If this is the case, the package

218

# src directory won't exist.

219

if not os.path.exists(src_dir):

220

self.src_dir = DEFAULT_SRC_DIR

221

self.built_files = []

222

self.skins = {}

223

if extra_files is None:

224

self.extra_files = []

225

else:

226

self.extra_files = extra_files

227

228

self.exclusion_regex = exclude_regex

229

self.file_type = file_type

230

self.copy_yui_to = copy_yui_to

231

232

self.log("Using filter: " + self.file_type)

233

234

def log(self, msg):

235

sys.stdout.write(msg + '\n')

236

237

def fail(self, msg):

238

"""An error was encountered, abort build."""

239

sys.stderr.write(msg + '\n')

240

sys.exit(1)

241

242

def file_is_excluded(self, filepath):

243

"""Is the given file path excluded from the rollup process?"""

244

if not self.exclusion_regex:

245

# Include everything.

246

return False

247

return re.search(self.exclusion_regex, filepath)

248

249

def ensure_build_directory(self, path):

250

"""Make sure that the named relative path is a directory."""

251

target_dir = os.path.join(self.build_dir, path)

252

if os.path.exists(target_dir):

253

if not os.path.isdir(target_dir):

254

self.fail(

255

"The target path, '%s', is not a directory!" % target_dir)

256

else:

257

self.log('Creating %s' % target_dir)

258

os.makedirs(target_dir)

259

return target_dir

260

261

def ensure_link(self, src, dst):

262

"""Make sure that src is linked to dst."""

263

if os.path.lexists(dst):

264

if not os.path.islink(dst):

265

self.fail(

266

"The target path, '%s', is not a symbolic link! " % dst)

267

else:

268

self.log('Linking %s -> %s' % (src, dst))

269

os.symlink(src, dst)

270

271

def link_and_minify(self, component, js_file):

272

"""Create raw, debug and min version of js_file."""

273

component_dir = os.path.join(self.build_dir, component)

274

basename = os.path.splitext(os.path.basename(js_file))[0]

275

276

raw_file = os.path.join(component_dir, basename + '.js')

277

rel_js_file = relative_path(raw_file, js_file)

278

self.ensure_link(rel_js_file, raw_file)

279

280

debug_file = os.path.join(component_dir, basename + '-debug.js')

281

self.ensure_link(rel_js_file, debug_file)

282

283

min_file = os.path.join(component_dir, basename + '-min.js')

284

if (not os.path.exists(min_file)

285

or os.stat(min_file).st_mtime < os.stat(js_file).st_mtime):

286

self.log("Minifying %s into %s." % (js_file, min_file))

287

js_in = open(js_file, 'r')

288

min_out = open(min_file, 'w')

289

minifier = JavascriptMinify()

290

minifier.minify(js_in, min_out)

291

js_in.close()

292

min_out.close()

293

294

self.built_files.append(

295

{'raw': raw_file,

296

'debug': debug_file,

297

'min': min_file})

298

299

def build_assets(self, component_name):

300

"""Build a component's "assets" directory."""

301

join = os.path.join

302

isdir = os.path.isdir

303

304

assets_path = join(component_name, 'assets')

305

src_assets_dir = join(self.src_dir, assets_path)

306

if not isdir(src_assets_dir):

307

return

308

309

target_assets_dir = self.ensure_build_directory(assets_path)

310

# Symlink everything except the skins subdirectory.

311

self.link_directory_content(

312

src_assets_dir, target_assets_dir,

313

lambda src: not src.endswith('skins'))

314

315

src_skins_dir = join(src_assets_dir, 'skins')

316

if not isdir(src_skins_dir):

317

return

318

319

# Process sub-skins.

320

for skin in os.listdir(src_skins_dir):

321

self.build_skin(component_name, skin)

322

323

def link_directory_content(self, src_dir, target_dir, link_filter=None):

324

"""Link all the files in src_dir into target_dir.

325

326

This doesn't recurse into subdirectories, but will happily link

327

subdirectories. It also skips linking backup files.

328

329

:param link_filter: A callable taking the source file as a parameter.

330

If the filter returns False, no symlink will be created. By

331

default a symlink is created for everything.

332

"""

333

for name in os.listdir(src_dir):

334

if name.endswith('~'):

335

continue

336

src = os.path.join(src_dir, name)

337

if link_filter and not link_filter(src):

338

continue

339

target = os.path.join(target_dir, name)

340

self.ensure_link(relative_path(target, src), target)

341

342

def build_skin(self, component_name, skin_name):

343

"""Build a skin for a particular component."""

344

join = os.path.join

345

346

skin_dir = join(component_name, 'assets', 'skins', skin_name)

347

src_skin_dir = join(self.src_dir, skin_dir)

348

target_skin_dir = self.ensure_build_directory(skin_dir)

349

350

# Link everything in there

351

self.link_directory_content(src_skin_dir, target_skin_dir)

352

353

# Holds all the combined files that are part of the skin

354

skin_files = self.skins.setdefault(skin_name, [])

355

356

# Create the combined core+skin CSS file.

357

for skin_file in glob(join(src_skin_dir, '*-skin.css')):

358

module_name = os.path.basename(skin_file)[:-len('-skin.css')]

359

360

target_skin_file = join(target_skin_dir, '%s.css' % module_name)

361

skin_files.append(target_skin_file)

362

363

# Combine files from the build directory so that

364

# relative paths are sane.

365

css_files = [

366

os.path.join(target_skin_dir, os.path.basename(skin_file))]

367

core_css_file = join(

368

self.src_dir, component_name, 'assets',

369

'%s-core.css' % module_name)

370

if os.path.exists(core_css_file):

371

css_files.insert(0, core_css_file)

372

373

combined_css = CSSComboFile(css_files, target_skin_file)

374

if combined_css.needs_update():

375

self.log('Combining %s into %s...' % (

376

", ".join(map(os.path.basename, css_files)),

377

target_skin_file))

378

combined_css.update()

379

380

def update_combined_js_file(self):

381

# Compile all the files in one JS file. Apply the filter to see

382

# which file extensions we should include.

383

build_file = os.path.join(self.build_dir, "%s.js" % self.name)

384

385

included_files = []

386

extra_files = [f for f in self.extra_files if f.endswith('.js')]

387

built_files = [f[self.file_type] for f in self.built_files]

388

389

included_files.extend(extra_files)

390

included_files.extend(built_files)

391

files_to_combine = [f for f in included_files

392

if not self.file_is_excluded(f)]

393

394

combined_js = JSComboFile(files_to_combine, build_file)

395

if combined_js.needs_update():

396

self.log('Updating %s...' % build_file)

397

combined_js.update()

398

399

def update_combined_css_skins(self):

400

"""Create one combined CSS file per skin."""

401

extra_css_files = [f for f in self.extra_files if f.endswith('.css')]

402

for skin_name in self.skins:

403

skin_build_file = os.path.join(self.build_dir, "%s-%s.css" %

404

(self.name, skin_name))

405

406

css_files = extra_css_files + self.skins[skin_name]

407

combined_css = CSSComboFile(css_files, skin_build_file)

408

if combined_css.needs_update():

409

self.log('Updating %s...' % skin_build_file)

410

combined_css.update()

411

412

def find_components(self):

413

"""Find all of the project sub-component names and directories."""

414

for name in os.listdir(self.src_dir):

415

path = os.path.join(self.src_dir, name)

416

if not os.path.isdir(path):

417

continue

418

yield name, path

419

420

def copy_yui_package(self):

421

"""Copy the contents of our 'yui' dir to self.copy_yui_to."""

422

yui_dirs = glob(os.path.join(self.src_dir, 'yui') + '/*')

423

for dir in yui_dirs:

424

dir_name = os.path.split(dir)[-1]

425

dst_dir = os.path.join(self.copy_yui_to, dir_name)

426

if not os.path.isdir(dst_dir):

427

shutil.copytree(dir, dst_dir)

428

429

def do_build(self):

430

self.copy_yui_package()

431

for name, cpath in self.find_components():

432

files_to_link = glob(os.path.join(cpath, '*.js'))

433

if len(files_to_link) == 0:

434

continue

435

self.ensure_build_directory(name)

436

437

for js_file in files_to_link:

438

self.link_and_minify(name, js_file)

439

440

self.build_assets(name)

441

442

self.update_combined_js_file()

443

self.update_combined_css_skins()

444

445

446

def get_options():

447

"""Parse the command line options."""

448

parser = optparse.OptionParser(

449

usage="%prog [options] [extra_files]",

450

description=(

451

"Create a build directory of CSS/JS files. "

452

))

453

parser.add_option(

454

'-n', '--name', dest='name', default='lazr',

455

help=('The basename of the generated compilation file. Defaults to '

456

'"lazr".'))

457

parser.add_option(

458

'-b', '--builddir', dest='build_dir', default=BUILD_DIR,

459

help=('The directory that should contain built files.'))

460

parser.add_option(

461

'-s', '--srcdir', dest='src_dir', default=PKG_SRC_DIR,

462

help=('The directory containing the src files.'))

463

parser.add_option(

464

'-x', '--exclude', dest='exclude', default='',

465

metavar='REGEX',

466

help=('Exclude any files that match the given regular expression.'))

467

parser.add_option(

468

'-f', '--filetype', dest='file_type', default='min',

469

help=('Only bundle files in the source directory that match the '

470

'specified file-type filter. Possible values are '

471

'[min, raw, debug]. [default: %default]'))

472

parser.add_option(

473

'-c', '--copy-yui-to', dest='copy_yui_to', default=BUILD_DIR,

474

help=('Copy the YUI tree to the given directory before building'))

475

return parser.parse_args()

476

477

478

def main():

479

options, extra= get_options()

480

Builder(

481

name=options.name,

482

build_dir=options.build_dir,

483

src_dir=options.src_dir,

484

extra_files=extra,

485

exclude_regex=options.exclude,

486

file_type=options.file_type,

487

copy_yui_to=options.copy_yui_to,

488

).do_build()