Commits

Enis Afgan committed 964290a Merge

Merge

Comments (0)

Files changed (9)

eggs.ini

File contents unchanged.

lib/galaxy/webapps/gc2/app.py

         return self.ami
         
     def get_type( self ):
+        if self.TESTFLAG is True:
+                return "TestType"
         if self.type is None:
             for i in range(0, 5):
                 try:

lib/galaxy/webapps/gc2/buildapp.py

     log.debug( "Enabling 'x-forwarded-host' middleware" )
     # Paste digest authentication
     svars = shellVars2Dict('userData.txt')
-    if 'PASSWORD' in svars:
+    if 'PASSWORD' in svars and svars['PASSWORD'] != '':
         from paste.auth.basic import AuthBasicHandler
         app = AuthBasicHandler(app, 'GC2 Administration', gc2_authfunc)
     return app

lib/galaxy/webapps/gc2/controllers/root.py

 
     @web.expose
     def instance_feed_json(self, trans):
-        dict_feed = {'instances' : [x.get_status_dict() for x in self.app.manager.worker_instances]}
+        dict_feed = {'instances' : [self.app.manager.get_status_dict()] + [x.get_status_dict() for x in self.app.manager.worker_instances]}
         return to_json_string(dict_feed)
     
     @web.expose

lib/galaxy/webapps/gc2/static/january_2008_style/blue/base.css

 
 #cluster_view_tooltip{
     margin-left:5px;
+    margin-top:2px;
     padding:2px;
     background:#FFFFFF;
-/*    border:1px solid #222222;*/
     float:right;
     width:110px;
     height:110px;
-
     color: #333;
     text-decoration: none;
     border: solid #aaaaaa 1px;

lib/galaxy/webapps/gc2/static/scripts/cluster_canvas.js

 }
 
 function renderGraph(){
-    // Ensure we have a context to start with...
     if (!ctx){
         return;
     }
-	// $('#status').append("Render Start. ");
 	ctx.clearRect(0, 0, c_width, c_height);
 	if (instances.length < 20){
-        // Default rendering.
 		var q = 0;
 		for ( i = 0 ; i < n_height; i++){
 			for ( j = 0; j < n_width; j++){
 				b_y = i*b_height + i*b_spacing;
 				b_xdx = b_x + b_width;
 				b_ydy = b_y + b_height;
-
 				if (q == instances.length){
 					// Drop shadow for boxes
 					ctx.fillStyle = "rgb(230, 230, 230)";
 					ctx.restore();
 					continue;
 				}
-				
 				// $('#status').append("Varcheck " + q);
 				instances[q].b_x = b_x;
 				instances[q].b_y = b_y;
 				instances[q].b_xdx = b_xdx;
 				instances[q].b_ydy = b_ydy;
-				
 				// Drop shadow for boxes
 				ctx.fillStyle = "rgb(230, 230, 230)";
 				roundedBox(x_offset + b_x+2, y_offset + b_y+2, b_width, b_height, b_corner_rad, ctx);
-
                 if (instances[q].ld != 0){
     				ld_arr = instances[q].ld.split(" ");
                 }else{
                     ld_arr = []
                 }
-			    if (instances[q].worker_status == 'Error'){
+                if (instances[q].worker_status == 'Error'){
 			        ctx.fillStyle = "#DF594B";
     				roundedBox(x_offset + b_x, y_offset +  b_y, b_width, b_height, b_corner_rad, ctx);
 			    }else if (ld_arr.length == 3){
 					ctx.fillStyle = "#575757";
     				roundedBox(x_offset + b_x, y_offset + b_y, b_width, b_height, b_corner_rad, ctx);
 				}
-                // else if(state == "Starting"){
-                //  ctx.fillStyle = "#E6FFA6";
-                //                  roundedBox(x_offset + b_x, y_offset + b_y, b_width, b_height, b_corner_rad, ctx);
-                // }
 				else{
                     // Else we have numbers to parse and display, instance is running.
 					ctx.fillStyle = "#FFDC40";
 
 function refreshTip(){
     if (selected_instance != -1 && selected_instance < instances.length){
-        // i_str = "<ul><li><b>" + instances[selected_instance][4] + "</b></li><li>" + instances[selected_instance][6] + "</li>\
-        i_str = "<ul><li><b>" + instances[selected_instance].id + "</b></li><li>State: " + instances[selected_instance].worker_status + "</li><li>Alive: " + instances[selected_instance].time_in_state + "</li>\
-        <li style='height:6px;'></li><li><div class='status_" + ARRAY_COLORS[1 + get_vol_ind(instances[selected_instance])] + "'>&nbsp;</div>Filesystems</li>\
-    	<li><div class='status_" + ARRAY_COLORS[1 + parseInt(instances[selected_instance].get_cert)] + "'>&nbsp;</div>Permissions</li>\
-    	<li><div class='status_" + ARRAY_COLORS[1 + parseInt(instances[selected_instance].sge_started)] + "'>&nbsp;</div>Scheduler</li>";
+        if (selected_instance == 0){
+            i_str = "<ul><li><b>Master Node</b></li><li>&nbsp;</li><li><b>" + instances[selected_instance].id + "</b></li><li>Alive: " + instances[selected_instance].time_in_state + "</li><li>Type: " + instances[selected_instance].instance_type + "</li>";
+        }
+        else{
+            i_str = "<ul><li><b>" + instances[selected_instance].id + "</b></li><li>State: " + instances[selected_instance].worker_status + "</li><li>Alive: " + instances[selected_instance].time_in_state + "</li>\
+            <li style='height:6px;'></li><li><div class='status_" + ARRAY_COLORS[1 + get_vol_ind(instances[selected_instance])] + "'>&nbsp;</div>Filesystems</li>\
+    	    <li><div class='status_" + ARRAY_COLORS[1 + parseInt(instances[selected_instance].get_cert)] + "'>&nbsp;</div>Permissions</li>\
+    	    <li><div class='status_" + ARRAY_COLORS[1 + parseInt(instances[selected_instance].sge_started)] + "'>&nbsp;</div>Scheduler</li>";
+	    }
         $('#cluster_view_tooltip').html(i_str);
     }else{
         $('#cluster_view_tooltip').html('');

lib/galaxy/webapps/gc2/templates/index.mako

         <span id="du-used">0</span> / <span id="du-total">0</span> (<span id="du-pct">0</span>) <span id='expand_vol' style='cursor: pointer;background-image:url(/cloud/static/images/disc_plus.png); background-repeat:no-repeat; display:inline-block; width:20px;'>&nbsp;</span>
     	##<span id="snap-status"></span><span id="snap-progress"></span>
 	</td></tr>
-    <tr><td><h4>Instance status: </h4></td><td>
+    <tr><td><h4>Worker status: </h4></td><td>
         <b>Idle</b>: <span id="status-idle">0</span>
         <b>Available</b>: <span id="status-available">0</span>
         <b>Requested</b>: <span id="status-total">0</span>

lib/galaxy/webapps/gc2/util/master.py

 """Galaxy GC2 master manager"""
-import logging, logging.config, threading, sys, os, time, subprocess, string, tempfile, re
+import logging, logging.config, threading, sys, os, time, subprocess, string, tempfile, re, commands
 import traceback, shutil, Queue, tarfile, pwd, grp, shutil, random, urllib2
 import datetime as dt
 
 from galaxy.util.bunch import Bunch
 
 from galaxy.webapps.gc2.util import misc, comm, introspect
+from galaxy.webapps.gc2.util.templates import *
 from galaxy.webapps.gc2.util.paths import *
 
 log = logging.getLogger( __name__ )
     SHUT_DOWN="SHUT_DOWN" # Because we don't really support cluster restart
  )
 
-sge_install_template = """
-SGE_ROOT="/opt/sge"
-SGE_QMASTER_PORT="6444"
-SGE_EXECD_PORT="6445"
-SGE_ENABLE_SMF="false"
-SGE_CLUSTER_NAME="GalaxyEC2"
-SGE_JMX_PORT=""
-SGE_JMX_SSL="false"
-SGE_JMX_SSL_CLIENT="false"
-SGE_JMX_SSL_KEYSTORE=""
-SGE_JMX_SSL_KEYSTORE_PW=""
-SGE_JVM_LIB_PATH=""
-SGE_ADDITIONAL_JVM_ARGS=""
-CELL_NAME="default"
-ADMIN_USER=""
-QMASTER_SPOOL_DIR="/opt/sge/default/spool/qmaster"
-EXECD_SPOOL_DIR="/opt/sge/default/spool/execd"
-GID_RANGE="20000-20100"
-SPOOLING_METHOD="classic"
-DB_SPOOLING_SERVER="none"
-DB_SPOOLING_DIR="/opt/sge/default/spooldb"
-PAR_EXECD_INST_COUNT="20"
-ADMIN_HOST_LIST="%s"
-SUBMIT_HOST_LIST="%s"
-EXEC_HOST_LIST="%s"
-EXECD_SPOOL_DIR_LOCAL=""
-HOSTNAME_RESOLVING="true"
-SHELL_NAME="ssh"
-COPY_COMMAND="scp"
-DEFAULT_DOMAIN="none"
-ADMIN_MAIL="none"
-ADD_TO_RC="false"
-SET_FILE_PERMS="true"
-RESCHEDULE_JOBS="wait"
-SCHEDD_CONF="1"
-SHADOW_HOST=""
-EXEC_HOST_LIST_RM=""
-REMOVE_RC="false"
-WINDOWS_SUPPORT="false"
-WIN_ADMIN_NAME="Administrator"
-WIN_DOMAIN_ACCESS="false"
-CSP_RECREATE="true"
-CSP_COPY_CERTS="false"
-CSP_COUNTRY_CODE="DE"
-CSP_STATE="Germany"
-CSP_LOCATION="Building"
-CSP_ORGA="Organisation"
-CSP_ORGA_UNIT="Organisation_unit"
-CSP_MAIL_ADDRESS="name@yourdomain.com"
-"""
-
-sge_host_conf_template = """
-hostname %s
-load_scaling NONE
-complex_values NONE
-user_lists NONE
-xuser_lists NONE
-projects NONE
-xprojects NONE
-usage_scaling NONE
-report_variables NONE
-
-"""
-
-all_q_template = """
-qname                 all.q
-hostlist              @allhosts
-seq_no                0
-load_thresholds       np_load_avg=1.75
-suspend_thresholds    NONE
-nsuspend              1
-suspend_interval      00:05:00
-priority              0
-min_cpu_interval      00:05:00
-processors            UNDEFINED
-qtype                 BATCH INTERACTIVE
-ckpt_list             NONE
-pe_list               make
-rerun                 FALSE
-slots                 1
-tmpdir                /mnt/galaxyData/tmp
-shell                 /bin/bash
-prolog                NONE
-epilog                NONE
-shell_start_mode      posix_compliant
-starter_method        NONE
-suspend_method        NONE
-resume_method         NONE
-terminate_method      NONE
-notify                00:00:60
-owner_list            NONE
-user_lists            NONE
-xuser_lists           NONE
-subordinate_list      NONE
-complex_values        NONE
-projects              NONE
-xprojects             NONE
-calendar              NONE
-initial_state         default
-s_rt                  INFINITY
-h_rt                  INFINITY
-s_cpu                 INFINITY
-h_cpu                 INFINITY
-s_fsize               INFINITY
-h_fsize               INFINITY
-s_data                INFINITY
-h_data                INFINITY
-s_stack               INFINITY
-h_stack               INFINITY
-s_core                INFINITY
-h_core                INFINITY
-s_rss                 INFINITY
-h_rss                 INFINITY
-s_vmem                INFINITY
-h_vmem                INFINITY
-"""
 
 class ConsoleManager( object ):
     def __init__( self, app ):
         self.disk_total = "0"
         self.disk_used = "0"
         self.disk_pct = "0%"
-
+        self.startup_time = dt.datetime.utcnow()
+        
         self.start_galaxy = False
         self.galaxy_starting = False
         
         self.snapshot_progress = None
         self.snapshot_status = None
 
+        self.gc_standalone = False
+
         if self.app.TESTFLAG is True:
             self.volumes['galaxyTools'] = [5, 'tool_device', False, None]
             self.volumes['galaxyIndices'] = [6, 'index_device', True, None]
 
         SGE_config_file = '%s/galaxyEC2.conf' % P_SGE_ROOT
         f = open( SGE_config_file, 'w' )
-        print >> f, sge_install_template % ( self.app.get_self_private_ip(), self.app.get_self_private_ip(), exec_nodes )
+        print >> f, SGE_INSTALL_TEMPLATE % ( self.app.get_self_private_ip(), self.app.get_self_private_ip(), exec_nodes )
         f.close()
         os.chown( SGE_config_file, pwd.getpwnam( "sgeadmin" )[2], grp.getgrnam( "sgeadmin" )[2] )
         log.debug( "Created SGE install template as file '%s'" % SGE_config_file )
             log.info("Successfully setup SGE; configuring SGE")
             SGE_allq_file = '%s/all.q.conf' % P_SGE_ROOT
             f = open( SGE_allq_file, 'w' )
-            print >> f, all_q_template
+            print >> f, ALL_Q_TEMPLATE
             f.close()
             os.chown( SGE_allq_file, pwd.getpwnam( "sgeadmin" )[2], grp.getgrnam( "sgeadmin" )[2] )
             log.debug( "Created SGE all.q template as file '%s'" % SGE_allq_file )
                 os.chown( host_conf_dir, pwd.getpwnam( "sgeadmin" )[2], grp.getgrnam( "sgeadmin" )[2] )
             host_conf = host_conf_dir + '/' + str( inst.id )
             f = open( host_conf, 'w' )
-            print >> f, sge_host_conf_template % ( inst_ip )
+            print >> f, SGE_HOST_CONF_TEMPLATE % ( inst_ip )
             f.close()
             os.chown( host_conf, pwd.getpwnam( "sgeadmin" )[2], grp.getgrnam( "sgeadmin" )[2] )
             log.debug( "Created SGE host configuration template as file '%s'." % host_conf )
             return False
         return True
 
+    def get_status_dict( self ):
+        num_cpus = int(commands.getoutput( "cat /proc/cpuinfo | grep processor | wc -l" ))
+        load = (commands.getoutput( "cat /proc/loadavg | cut -d' ' -f1-3" )).strip() # Returns system load in format "0.00 0.02 0.39" for the past 1, 5, and 15 minutes, respectivley
+        if load != 0:
+            lds = load.split(' ')
+            if len(lds) == 3:
+                load = "%s %s %s" % (float(lds[0]) / int(num_cpus), float(lds[1]) / int(num_cpus), float(lds[2]) / int(num_cpus))
+            else:
+                # Debug only, this should never happen.  If the interface is able to display this, there is load.
+                load = "0 0 0"
+        return  {'id' : self.app.get_instance_id(), 'ld' : load, 'time_in_state' : misc.formatDelta(dt.datetime.utcnow() - self.startup_time), 'instance_type' : self.app.get_type() }
+
 
 class ConsoleMonitor( object ):
     def __init__( self, app ):
         self.sge_started = 0
         self.worker_status = 'Pending'
         self.load = 0
+        self.type = 'Unknown'
     
     def get_status_dict( self ):
         toret = {'id' : self.id, 
                  'get_cert' : self.get_cert, 
                  'sge_started' : self.sge_started, 
                  'worker_status' : self.worker_status,
-                 'instance_state' : self.m_state}
+                 'instance_state' : self.m_state,
+                 'instance_type' : self.type}
 
         if self.load != 0:
             lds = self.load.split(' ')

lib/galaxy/webapps/gc2/util/templates.py

+SGE_INSTALL_TEMPLATE = """
+SGE_ROOT="/opt/sge"
+SGE_QMASTER_PORT="6444"
+SGE_EXECD_PORT="6445"
+SGE_ENABLE_SMF="false"
+SGE_CLUSTER_NAME="GalaxyEC2"
+SGE_JMX_PORT=""
+SGE_JMX_SSL="false"
+SGE_JMX_SSL_CLIENT="false"
+SGE_JMX_SSL_KEYSTORE=""
+SGE_JMX_SSL_KEYSTORE_PW=""
+SGE_JVM_LIB_PATH=""
+SGE_ADDITIONAL_JVM_ARGS=""
+CELL_NAME="default"
+ADMIN_USER=""
+QMASTER_SPOOL_DIR="/opt/sge/default/spool/qmaster"
+EXECD_SPOOL_DIR="/opt/sge/default/spool/execd"
+GID_RANGE="20000-20100"
+SPOOLING_METHOD="classic"
+DB_SPOOLING_SERVER="none"
+DB_SPOOLING_DIR="/opt/sge/default/spooldb"
+PAR_EXECD_INST_COUNT="20"
+ADMIN_HOST_LIST="%s"
+SUBMIT_HOST_LIST="%s"
+EXEC_HOST_LIST="%s"
+EXECD_SPOOL_DIR_LOCAL=""
+HOSTNAME_RESOLVING="true"
+SHELL_NAME="ssh"
+COPY_COMMAND="scp"
+DEFAULT_DOMAIN="none"
+ADMIN_MAIL="none"
+ADD_TO_RC="false"
+SET_FILE_PERMS="true"
+RESCHEDULE_JOBS="wait"
+SCHEDD_CONF="1"
+SHADOW_HOST=""
+EXEC_HOST_LIST_RM=""
+REMOVE_RC="false"
+WINDOWS_SUPPORT="false"
+WIN_ADMIN_NAME="Administrator"
+WIN_DOMAIN_ACCESS="false"
+CSP_RECREATE="true"
+CSP_COPY_CERTS="false"
+CSP_COUNTRY_CODE="DE"
+CSP_STATE="Germany"
+CSP_LOCATION="Building"
+CSP_ORGA="Organisation"
+CSP_ORGA_UNIT="Organisation_unit"
+CSP_MAIL_ADDRESS="name@yourdomain.com"
+"""
+
+SGE_HOST_CONF_TEMPLATE = """
+hostname %s
+load_scaling NONE
+complex_values NONE
+user_lists NONE
+xuser_lists NONE
+projects NONE
+xprojects NONE
+usage_scaling NONE
+report_variables NONE
+
+"""
+
+ALL_Q_TEMPLATE = """
+qname                 all.q
+hostlist              @allhosts
+seq_no                0
+load_thresholds       np_load_avg=1.75
+suspend_thresholds    NONE
+nsuspend              1
+suspend_interval      00:05:00
+priority              0
+min_cpu_interval      00:05:00
+processors            UNDEFINED
+qtype                 BATCH INTERACTIVE
+ckpt_list             NONE
+pe_list               make
+rerun                 FALSE
+slots                 1
+tmpdir                /mnt/galaxyData/tmp
+shell                 /bin/bash
+prolog                NONE
+epilog                NONE
+shell_start_mode      posix_compliant
+starter_method        NONE
+suspend_method        NONE
+resume_method         NONE
+terminate_method      NONE
+notify                00:00:60
+owner_list            NONE
+user_lists            NONE
+xuser_lists           NONE
+subordinate_list      NONE
+complex_values        NONE
+projects              NONE
+xprojects             NONE
+calendar              NONE
+initial_state         default
+s_rt                  INFINITY
+h_rt                  INFINITY
+s_cpu                 INFINITY
+h_cpu                 INFINITY
+s_fsize               INFINITY
+h_fsize               INFINITY
+s_data                INFINITY
+h_data                INFINITY
+s_stack               INFINITY
+h_stack               INFINITY
+s_core                INFINITY
+h_core                INFINITY
+s_rss                 INFINITY
+h_rss                 INFINITY
+s_vmem                INFINITY
+h_vmem                INFINITY
+"""