Source

buildplace / script / build_source.py

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
#!/usr/bin/python3
# Copyright (c) 2012 All Right Reserved, Wisut Hantanong
#
# This file is part of buildplace.
#
# buildplace is free software: you can redistribute it and/or modify
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# at your option) any later version.
#
# buildplace is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with buildplace.  If not, see <http://www.gnu.org/licenses/>.

import sys, os, yaml, logging, tkinter, time, subprocess

#FIXME:HARDCODE: get terminal dimension from system instead of hard-coding
# btw terminal is use for debug purpose only
TERM_WIDTH = 150

# global variables for inter-function communication
logger = logging.getLogger('buildplace') # logger for debuging
ENV = {} # global environ, we use this modified ENV in addition of per-source envs and system os.environ
CONFIGS = {} # program configuration
# TODO:try to minimize value passing via UI by using CONFIGS when applicable, this will help in UI redesign process
UIS = {} # for cross-function UI update
APP = 0 # tk app reference

# script entry point
def main():
	# DEBUG: clear console for easy debug message reading
	os.system('clear')

	global CONFIGS # need this to tell python that we will write to global CONFIGS 
	global UIS # need this too for modifying the global UIS

	CONFIGS = loadconfig() # we only want the log_dir
	if CONFIGS == {}:
		quit() # nothing to do if there is no valid configuration

	logger.debug('CONFIGS {}'.format(CONFIGS))

	print('Looking for log_dir in config file')
	log_dir = CONFIGS['common']['log_dir']

	initlogger(log_dir)

	logger.debug('main: logger init')
	logger.debug('CONFIGS: {0}'.format(CONFIGS))

	create_main_window()

	# application main loop
	APP.mainloop()

	#inject_default_config(configs)

	#save_config(configs)

	logger.debug('main: exit')
	return

# this enable config editing without script restart 
def reload_config():
	global CONFIGS
	global UIS

	CONFIGS = loadconfig() 

	# clear and fill listbox with entry in config file
	UIS['sourcelistbox'].delete(0, tkinter.END)
	UIS['status'].set('Nothing selected')
	for w in UIS['source_detail'].winfo_children():
		w.destroy()

	print('source')
	sources = {}
	for src in CONFIGS['source'] if 'source' in CONFIGS else {}:
		sources[src] = CONFIGS['source'][src] # get source sub-item
		print('name {0} version {1}'.format(src, sources[src]['version']))

	for src in sources:
		UIS['sourcelistbox'].insert(tkinter.END, src)

	return

# create main tk window with source selection listbox filled
def create_main_window():
	global APP # this prevent APP being distroy after function exit

	#FIXME: BAD widgets layout
	# I need to invest in 'how to write a good tkinter'
	#
	# |----------------------------------|
	# |             status               |
	# |----------------------------------|
	# | source        | cout             |
	# |               -------------------|
	# |               | cerr             |
	# |----------------------------------|
	# | detail and controls              |
	# |----------------------------------|
	#
	# main window
	APP = tkinter.Tk()

	# main frame
	main_frame = tkinter.Frame(APP, borderwidth=1)
	main_frame.pack(fill=tkinter.BOTH, expand=1)

	# main label
	tkinter.Label(main_frame, text='buildplace 0.0.1').pack()

	# config reload button
	reload_config_btn = tkinter.Button(main_frame, text='reload config', command=reload_config)
	reload_config_btn.pack(side=tkinter.TOP, padx=5, pady=5)

	# current status lable
	UIS['status'] = tkinter.StringVar()
	tkinter.Label(main_frame, textvariable=UIS['status']).pack()

	# left frame
	UIS['frame_left'] = tkinter.Frame(main_frame, borderwidth=1)
	UIS['frame_left'].pack(side=tkinter.LEFT, fill=tkinter.BOTH, expand=1)

	# right frame
	UIS['frame_right'] = tkinter.Frame(main_frame, borderwidth=1)
	UIS['frame_right'].pack(side=tkinter.RIGHT, fill=tkinter.BOTH, expand=1)

	#  source frame (for listbox)
	UIS['source_frame'] = tkinter.Frame(UIS['frame_left'], borderwidth=1)
	UIS['source_frame'].pack(fill=tkinter.BOTH, expand=1)

	#  source frame (for detail)
	UIS['source_detail'] = tkinter.Frame(UIS['frame_left'], borderwidth=1)
	UIS['source_detail'].pack(fill=tkinter.BOTH, expand=1)

	# source selection listbox
	UIS['sourcelistbox'] = tkinter.Listbox(UIS['source_frame'])

	reload_config()

	UIS['sourcelistbox'].bind('<<ListboxSelect>>', update_source_detail)
	UIS['sourcelistbox'].pack(fill=tkinter.BOTH, expand=1)
	return

# query for details of a source
def update_source_detail(event):
	logger.debug('update_source_detail: enter')

	global UIS # we modify UIS['cur_source'] here

	lbx = event.widget # sender widget is listbox
	index = int(lbx.curselection()[0]) # get current selection index (no multipleselction)
	selection = lbx.get(index) # selection value
	print('listbox selected index {}, value {}'.format(index, selection))

	logger.debug('CONFIGS: {0}'.format(CONFIGS))
	for w in UIS['source_detail'].winfo_children(): # clear source_detail frame
		w.destroy()

	cur_source = CONFIGS['source'][selection] # get source item from selection key
	UIS['cur_source'] = cur_source # update global current selected source
	cur_source['name'] = selection 

	UIS['status'].set('{0} {1}'.format(selection, cur_source['version']))

	# show configuration arguments in source detail listbox 
	tkinter.Label(UIS['source_detail'], text=cur_source['config_cmd'] if 'config_cmd' in cur_source else 'N/A').pack()

	config_opts = tkinter.Listbox(UIS['source_detail'])
	if 'config_args' in cur_source:
		if cur_source['config_args']:
			for opt in cur_source['config_args']:
				config_opts.insert(tkinter.END, opt)
	config_opts.pack(fill=tkinter.BOTH, expand=1)
	config_opts.config(state=tkinter.DISABLED) # not support editing yet

	tkinter.Label(UIS['source_detail'], text=cur_source['build_cmd'] if 'build_cmd' in cur_source else 'N/A').pack()
	build_opts = tkinter.Listbox(UIS['source_detail'])
	if 'build_args' in cur_source:
		if cur_source['build_args']:
			for opt in cur_source['build_args']:
				build_opts.insert(tkinter.END, opt)
	build_opts.pack(fill=tkinter.BOTH, expand=1)
	build_opts.config(state=tkinter.DISABLED) # not support editing yet

	config_btn = tkinter.Button(UIS['source_detail'], text='configure', command=config_source)
	config_btn.pack(side=tkinter.RIGHT, padx=5, pady=5)

	make_btn = tkinter.Button(UIS['source_detail'], text='build', command=build_source)
	make_btn.pack(side=tkinter.RIGHT, padx=5, pady=5)

	logger.debug('update_source_detail: exit')
	return

def build_source():
	logger.debug('build_source: enter')

	cur_source = UIS['cur_source'] # get current source to work with
	print('building ... {}-{}'.format(cur_source['name'], cur_source['version']))

	source_name = cur_source['name'] + '-' + cur_source['version']
	# build_dir only valid after configurator called
	if 'build_dir' in cur_source:
		build_dir = cur_source['build_dir']
	else:
		print('build_dir not valid')
		#UIS['status'] = 'N/A'
		return

	# if build_cmd is not provide we do nothing
	if('build_cmd' in cur_source):
		build_cmd = cur_source['build_cmd']
	else:
		return

	# if build_args is not provide we don't need to expand it
	build_args = []
	if 'build_args' in cur_source:
		raw_build_args= cur_source['build_args']

		# expand env in config_args
		if raw_build_args:
			for arg in raw_build_args:
				for i in range(10):
					arg = os.path.expandvars(arg)
				build_args.append(arg)
				#print('bld ', arg)

	print('source_name ', source_name)
	print('build_dir ', build_dir)
	print('build_cmd ', build_cmd)
	print('build_args ', build_args)

	ret = -1
	args = {}

	build_env = {}
	for key, val in os.environ.items(): 
		build_env[key] = val # get system envs (include common env by loadconfig())

	if ENV: 
		print('common envs')
		for key, val in ENV.items(): 
			print(key, val)

	source_env = cur_source['envs'] if 'envs' in cur_source else []
	if source_env:
		print('source envs')
		for env in cur_source['envs']:
			key, val = env.split('=')
			for i in range(10): 
				val = os.path.expandvars(val)
			build_env[key] = val # add source env to config_env
			print(key, ' ' , val)
	else:
		print('No source envs')
		
	print('using builder "{}"'.format(build_cmd))
	args['task_name'] = source_name
	args['env'] = build_env
	args['working_dir'] = build_dir
	args['command'] = build_cmd
	args['arguments'] = build_args
	ret = do_exec(args)

	if ret == 0:
		print('INFO: build success ({0})'.format(ret))
		UIS['status'].set('INFO: configure success ({0})'.format(ret))
		pass
	else:
		print('ERROR: configure failed ({0})'.format(ret));
		UIS['status'].set('ERROR: configure failed ({0})'.format(ret))
		pass

	logger.debug('build_source: exit')
	return


def config_source():
	logger.debug('config_source: enter')
	global UIS # we modify UIS['cur_source']

	cur_source = UIS['cur_source'] # get current source to work with
	print('configuring ... {}-{}'.format(cur_source['name'], cur_source['version']))

	source_name = cur_source['name'] + '-' + cur_source['version']
	source_dir = os.path.join(CONFIGS['common']['source_dir'], source_name)
	build_dir = os.path.join(CONFIGS['common']['build_dir'], source_name)
	for i in range(10):
		source_dir = os.path.expandvars(source_dir)
		build_dir = os.path.expandvars(build_dir)

	# if config_cmd is not provide we do nothing
	if('config_cmd' in cur_source):
		config_cmd = cur_source['config_cmd']
	else:
		return

	# if config_args is not provide we don't need to expand it
	config_args = []
	if 'config_args' in cur_source:
		raw_config_args= cur_source['config_args']

		# expand env in config_args
		if raw_config_args:
			for arg in raw_config_args:
				for i in range(10):
					arg = os.path.expandvars(arg)
				config_args.append(arg)
				#print('cfg ', arg)

	config_env = {}
	for key, val in os.environ.items():
		config_env[key] = val # get system envs (include common env by loadconfig())

	if ENV: 
		print('common envs')
		for key, val in ENV.items(): 
			print(key, val)

	source_env = cur_source['envs'] if 'envs' in cur_source else []
	if source_env:
		print('source envs')
		for env in cur_source['envs']:
			key, val = env.split('=')
			for i in range(10): 
				val = os.path.expandvars(val)
			config_env[key] = val # add source env to config_env
			print(key, ' ' , val)
	else:
		print('No source envs')
	
	# configuring selection by type of configurator
	ret = -1
	args = {}
	if config_cmd[0] == '.': # local configurator, e.g. ./configure, ./autogen.sh
		print('using local configurator "{}"'.format(config_cmd))
		cur_source['build_dir'] = source_dir # pass build_dir to builder

		args['task_name'] = source_name
		args['env'] = config_env
		args['working_dir'] = source_dir
		args['command'] = config_cmd
		args['arguments'] = config_args
		ret = do_exec(args)
	else: # system wide, e.g. cmake
		print('using system configurator "{}"'.format(config_cmd))
		if not os.path.exists(build_dir):
			os.makedirs(build_dir)
		cur_source['build_dir'] = build_dir # pass build_dir to builder 

		cmakecache = os.path.join(build_dir, 'CMakeCache.txt')
		if os.path.exists(cmakecache):
			print('removing CMakeCache.txt')
			os.remove(cmakecache)
			

		print('source_name ', source_name)
		print('source_dir ', source_dir)
		print('build_dir ', build_dir)
		print('config_cmd ', config_cmd)
		print('config_args ', config_args)

		args['task_name'] = source_name
		args['env'] = config_env
		args['working_dir'] = build_dir
		args['command'] = config_cmd
		args['arguments'] = [source_dir] + config_args
		ret = do_exec(args)

	if ret == 0:
		print('INFO: configure success ({0})'.format(ret))
		UIS['status'].set('INFO: configure success ({0})'.format(ret))
		pass
	else:
		print('ERROR: configure failed ({0})'.format(ret));
		UIS['status'].set('ERROR: configure failed ({0})'.format(ret))
		pass

	logger.debug('config_source: exit')
	return

#def do_make(args):
#	logger.debug('linux_do_make enter')
#
#	source = args['source']
#	source_dir = args['source_dir']
#	build_dir = args['build_dir']
#	install_dir = args['install_dir']
#	opts= args['opts'] if 'opts' in args else []
#	reset = args['reset'] if 'reset' in args else False
#	logger.info('linux_do_make args {0}'.format(args))
#
#	make_opts=['install'] + opts
#
#	# check if Makefile exists? 
#	#makefile=os.path.join(build_dir,"Makefile")
#	makefile=os.path.join(build_dir,"build.ninja")
#	if not os.path.exists(makefile):
#		print("ERROR: Makefile or build.ninja missing from ", build_dir)
#		UIS['status'].set('ERROR: notthin to build in {0}'.format(build_dir))
#		return -1
#
#	# call cmake for build Makefile 
#	make_args = {}
#	make_args['name'] = source
#	#make_args['cmd'] = 'make'
#	make_args['cmd'] = 'ninja'
#	make_args['wd'] = build_dir
#	make_args['cmd_args'] = make_opts
#
#	ret = do_exec(make_args)
#	if ret == 0:
#		print('INFO: make success ({0})'.format(ret))
#		UIS['status'].set('INFO: make success ({0})'.format(ret))
#		pass
#	else:
#		print('ERROR: make failed ({0})'.format(ret));
#		UIS['status'].set('ERROR: make failed ({0})'.format(ret))
#		pass
#
#	logger.debug('do_make exit')
#	return ret
#
#
#def do_cmake(args):
#	logger.debug('do_cmake enter')
#
#	source = args['source']
#	source_dir = args['source_dir']
#	build_dir = args['build_dir']
#	install_dir = args['install_dir']
#	opts= args['opts']
#	reset = args['reset'] if 'reset' in args else False
#	logger.info('do_cmake args {0}'.format(args))
#
#	cmake_opts=[ '-D{0}'.format(i) for i in opts]
#
#	# check for existing build path ,if not create one
#	if not os.path.exists(build_dir):
#		print("directory created ", build_dir)
#		os.makedirs(build_dir)
#	else: 
#		print("directory exists ", build_dir)
#
#	# delete CMakeCache.txt if exists? 
#	cache=os.path.join(build_dir,"CMakeCache.txt")
#	if reset:
#		print("Not using ", cache)
#		if os.path.exists(cache):
#			os.remove(cache)
#	else:
#		print("Using ", cache)
#		pass
#
#	# call cmake for generate Makefile 
#	cmake_args = {}
#	cmake_args['name'] = source
#	cmake_args['cmd'] = 'cmake'
#	cmake_args['wd'] = build_dir
#	# cmake_args['cmd_args'] = [source_dir] + ['-G','CodeBlocks - Unix Makefiles'] + cmake_opts
#	cmake_args['cmd_args'] = [source_dir] + ['-GNinja'] + cmake_opts 
#	#cmake_args['cmd_args'] = [source_dir] + ['-GUnix Makefiles'] + cmake_opts 
#
#	ret = do_exec(cmake_args)
#	if ret == 0:
#		print('INFO: cmake success ({0})'.format(ret))
#		UIS['status'].set('INFO: cmake success ({0})'.format(ret))
#		pass
#	else:
#		print('ERROR: cmake failed ({0})'.format(ret));
#		UIS['status'].set('ERROR: cmake failed ({0})'.format(ret))
#		pass
#
#	logger.debug('do_cmake exit')
#	return ret

def initlogger(log_dir):
	print('initlogger: raw log_dir=', log_dir)

	print('initlogger: full log_dir=', log_dir)
	if not os.path.exists(log_dir):
		os.makedirs(log_dir)

	hdlr=logging.FileHandler(os.path.join(log_dir, 'buildplace.py.log'))
	formatter=logging.Formatter('%(asctime)s %(levelname)s %(message)s')
	hdlr.setFormatter(formatter)
	logger.addHandler(hdlr)
	logger.setLevel(logging.DEBUG)

	logger.debug('initlogger exit')

'''
execute a process
WARNING: if shell=False any IO redirection in command migth fail
'''
def do_exec(args):
	for w in UIS['frame_right'].winfo_children():
		w.destroy()
	UIS['output_text'] = tkinter.Text(UIS['frame_right'])
	UIS['output_text'].pack(fill=tkinter.BOTH, expand=1) 
	UIS['error_text'] = tkinter.Text(UIS['frame_right'])
	UIS['error_text'].pack(fill=tkinter.BOTH, expand=1)

	logger.debug('do_exec enter')
	logger.debug('do_exec: args {0}'.format(args))

	ts=time.clock()

	name = args['task_name'] # use for log naming
	cmd = args['command'] # the actual command without arguments eg. 'cmake'
	cmd_args = args['arguments'] # list of command argument eg. '['x', 'y', 'z']
	wd = args['working_dir'] # working directory
	if not os.path.exists(wd):
		print('WARNING: working directory {} not exists. This migth cause configuring fail'.format(wd))
	env = args['env']

	log_dir = CONFIGS['common']['log_dir']
	cout_log=os.path.join(log_dir, name+ '-' + cmd.replace('./','') + '-cout.log')
	cerr_log=os.path.join(log_dir, name+ '-' + cmd.replace('./','') + '-cerr.log')

	cout=open(cout_log, 'wb')
	cerr=open(cerr_log, 'wb')

	command = [cmd] + cmd_args

	logger.info('command {0}'.format(command))
	print('command ', command)
	print('wd ', wd)
	print('env ')
	for key, val in env.items():
		print('   {} - {}'.format(key, val))
	print('command text', ' '.join(command))
	ret=subprocess.Popen(command, env=env, shell=False, cwd=wd, stdout=subprocess.PIPE, stderr=cerr)

	pad=' ' * (TERM_WIDTH + 5)
	while True:
		cout_line=ret.stdout.readline()
		if cout_line:
			#print(pad, end='\r')
			#print('==', cout_line.decode('utf-8')[0:TERM_WIDTH].replace('\n','\r'), end='\r')
			UIS['status'].set(cout_line.decode('utf-8')[0:TERM_WIDTH])
			UIS['frame_right'].update_idletasks()
			cout.write(cout_line)
		else:
			break
	print()

	ret.wait()

	cout.close()
	cerr.close()


	cout=open(cout_log)
	for i in cout.readlines():
		UIS['output_text'].insert(tkinter.END, i)

	cerr=open(cerr_log)
	for i in cerr.readlines():
		UIS['error_text'].insert(tkinter.END, i)

	cerr.close()
	cout.close()
	if os.stat(cout_log).st_size == 0: os.remove(cout_log)
	if os.stat(cerr_log).st_size == 0: os.remove(cerr_log)

	te=time.clock() - ts
	if ret.returncode != 0:
		logger.error('cmd return %d time %s', ret.returncode, te)
	else:
		logger.info('cmd return %d time %s', ret.returncode, te)

	logger.debug('do_exec exit')
	return ret.returncode

# load yaml configuration
def loadconfig(filename=''):
	global ENV # this function modify global ENV
	config = {}

	# default buildplace.conf in this script folder if filename is not specify
	if(filename == ''):
		filename = os.path.dirname(os.path.abspath(__file__)) + '/build_source.conf'

	print('using ', filename)

	try:
		config_file = open(filename)
		config = yaml.load(config_file)

		# set common env
		for env in config['common']['envs']:
			key, val = env.split('=')
			for i in range(10): 
				val = os.path.expandvars(val)
			ENV[key] = val
			os.environ[key] = val # also pass common env to system (for used in var expansion)
			#print(key,'=', val)

		# expand all vars in common section, let system handle vars in command argument 
		# this allow ~/xxx (or ${env}/xxx in future) in config file
		for i in range(10): #FIXME:ASSUMPTION: env should not nest this deep
			config['common']['source_dir']	= os.path.expandvars(config['common']['source_dir'])
			config['common']['build_dir']	= os.path.expandvars(config['common']['build_dir'])
			config['common']['log_dir']	= os.path.expandvars(config['common']['log_dir'])
	except:
		print('ERROR: can\'t open configuration file \'{0}\''.format(os.path.join(os.getcwd(), filename)))
		print('       Please check that file exists and YAML-valid')
		print(sys.exc_info())
	finally:
		config_file.close()
		#print(config)

	return config


# python trick for entering main when calling ./build_source.py from shell
if __name__ == '__main__': main()