Source

Portfolio / gnib / load.pl

#!/bin/env perl

############################
## Author: Adam M Dutko
## License: GPL v3 or later
## 

## TODO
## 1) Need to do checks for exists
## 2) Add more params to accomodate other projects

use DBI;
use Data::Dumper;

############
## CONFIG ##
############
my $_NAME = $0;  
my $_VERSION = "0.01";
my $_BASE = "logs/";
my $_STATS_FILE = $_BASE . "gnib.csv";
my $_DB_USER = "gnibAdmin";
my $_DB_HOST = "localhost";
my $_DB_PASS = "gnibs";
my $_DB = "gnib";

###########
## DATABASE
##
my $dsn = "DBI:mysql:$_DB;host=$_DB_HOST";
my $_DB_HANDLE = DBI->connect($dsn,$_DB_USER,$_DB_PASS) or die("ERROR: Cannot connect.\n");


##############
## ROUTINES ##
##############

## create project record
## - check if not exists
## - create if not exists
## - return project id
sub create_project
{
    my $project_name = shift;
    my $_dbh = shift;

    ## Add project    
    my $project_stmt = "insert into projects (name) values (?);"; 
    my $_proj_sth = $_dbh->prepare($project_stmt);
    $_proj_sth->execute($project_name);
    
    my $project_id = $_dbh->{ q{mysql_insertid}};
    return $project_id;
} 

## create version record
## - check if not exists
## - create if not exists
## - return version id
sub create_version
{
    my $project_id = shift; 
    my $_dbh = shift;
    my $version = shift;

    ## Add version    
    my $version_stmt = "insert into versions (project_id,version) values (?,?);"; 
    my $_vers_sth = $_dbh->prepare($version_stmt);
    $_vers_sth->execute($project_id,$version);
    
    my $version_id = $_dbh->{ q{mysql_insertid}};
    return $version_id;
} 

## create statistic record
## - check if not exists
## - create if not exists
sub create_statistics
{
    my $version_info = shift;
    my %version_data = %{$version_info};
    my $_dbh = shift;
    my $_stat_file = shift;

    die ("ERROR: Statistics file not generated.\n") if (! -e $_stat_file);
    open(STAT_HANDLE,$_stat_file);    

    #print Dumper(%{$version_info});
    #exit;
 
    while(<STAT_HANDLE>)
    {
        my $_line = $_;

        ## skip first line
        next if $_line =~ /ARTIFACT/;

        ## skip blank lines
        next if $_line =~ / /;

        my $_found_version;
        my $_found_line;
        my @_versions = keys %version_data;
        #print "FOUND " . scalar @_versions . " versions.\n";
        foreach my $_version (@_versions)
        {

            #print "Looking at $_version and $_line.\n";
            my $_key_version = $_version;
            if ($_version eq "base")
            {
                $_version = "108";
            } else {
                $_version = join(".",split("",$_version));
            }

            my @_intermediate_split = split(",",$_line);
            my $_intermediate_version_full = $_intermediate_split[0]; 
            my $_intermediate_version_full_sub = substr($_intermediate_version_full,6); 
            ## NOTE: Archives need to be "tarred" with .tar. in the name.
            ## TODO: Abstract this away...
            my @_intermediate_version_parts = split(".tar.",$_intermediate_version_full_sub);
            pop(@_intermediate_version_parts);
            my $_intermediate_version;
            #print @_intermediate_version_parts . "\n";
            $_intermediate_version = join(".",@_intermediate_version_parts);
            #print "VERSION: " . $_key_version . " INTERVERSION: " . $_intermediate_version . "\n"; 

            if ($_key_version eq "base")
            {
                $_key_version = "108"; 
            }

            if ($_intermediate_version eq $_key_version)
            {
                $_found_version = $_key_version;
                $_found_line = $_line;
        	my @_stats = split(",",$_found_line);
        	my $version_id = $version_data{$_found_version};
                
                ## Remove me ... php specific
                if ($version_id eq "")
                {
                    $version_id = $version_data{"base"};
                }

        	my $size = $_stats[1];
       	 	my $c_count = $_stats[2];
        	my $h_count = $_stats[3];
        	my $cpp_count = $_stats[4];
        	my $dir_count = $_stats[5];
        	my $c_incl_count = $_stats[6];
        	my $h_incl_count = $_stats[7];

                my $_check_stat_id;

                ## check if file exists
                my $statistics_exists_stmt = "select id from versions where
                                              version = ?";
                my $_statistics_exists_sth = $_dbh->prepare($statistics_exists_stmt);
                $_statistics_exists_sth->execute($version_data{$_key_version});

                ## number of results
                my $result = $_statistics_exists_sth->rows;

                ## if statistics do not exist insert data
                print "VERSION ROWS: $result\n" if ($result != 0);
                if ($result == 0)
                {

        	    ## Add statistics
        	    my $statistic_stmt = "insert into statistics
                        	            (version_id,size,c_file_count,h_file_count,
                          		      cpp_file_count,directory_count,c_includes_count,
                       	        	      h_includes_count) values (?,?,?,?,?,?,?,?);";
        	    my $_stat_sth = $_dbh->prepare($statistic_stmt);
        	    $_stat_sth->execute($version_id,$size,$c_count,
               	           		     $h_count,$cpp_count,$dir_count,
               	        	             $c_incl_count,$h_incl_count);
                }
            }
        } 

    }
    close(STAT_HANDLE);
}

## create file record
## - check if not exists
## - create if not exists
## - return file id
sub create_file
{
    my $version_id = shift;
    my $_dbh = shift;
    my $_file_name = shift;

    #print "FILENAME: $_file_name \n";

    my $file_id;

    ## check if file exists
    my $file_exists_stmt = "select id from files where 
                            version_id = ? and name = ?";
    my $_file_exists_sth = $_dbh->prepare($file_exists_stmt);
    $_file_exists_sth->execute($version_id,$_file_name);
    
    ## number of results
    my $result = $_file_exists_sth->rows;

    ## if file does not exist insert data and return file_id
    if ($result == 0)
    {  
        my $add_file_stmt = "insert into files (version_id,name) values (?,?)";
        my $_add_file_sth = $_dbh->prepare($add_file_stmt);

        ## This only occurs when we have a weird result in the file due to
        ## line ending
        #die("ERROR: $_file_name") if ($_file_name eq "");

        $_add_file_sth->execute($version_id,$_file_name); 
        $file_id = $_dbh->{ q{mysql_insertid}}; 
    } else {
        my $_file_data = $_file_exists_sth->fetchall_hashref('id');
        my @ids = keys %{$_file_data};
        $file_id = $ids[0];
    }

    return $file_id;
}

## create component record
## - check if not exists
## - create if not exists
sub create_component
{
    my $file_id = shift;
    my $_dbh = shift;
    my $_component_name = shift;
    my $_component_type = shift;
    my $_component_line_number = shift;
    my $_component_value = shift;
    
    my $add_comp_stmt = "insert into components (file_id,name,type,line_number,
                                                value) values (?,?,?,?,?)"; 
    my $_add_comp_sth = $_dbh->prepare($add_comp_stmt);
    $_add_comp_sth->execute($file_id,$_component_name,$_component_type,
                            $_component_line_number,$_component_value);

}

## get all versions of the project
## - list contents of log directory for *.functions
## - parse filenames and pull out version
## - return list of versions
sub get_versions
{
    my $_file_ref = shift;
    my @_files = @{$_file_ref};

    my @versions;
    foreach my $_file (@_files)
    {
        my @first_parts = split("-",$_file);
        my @second_parts = split(".functions",$first_parts[1]); 
        
        ## Accomodate just php
        if (scalar @second_parts == 2)
        {
            push(@versions, $second_parts[0]);
        } else {
            push(@versions, "base");
        }
    } 
    
    return \@versions; 
}

################
## MAIN LOGIC ##
################
sub main
{
    my $project = shift;
    my $_dbh = shift;
    my $_base = shift;
    my $_stats_file = shift;

    my @function_files = `ls -1 $_base/*.functions`;

    ## 1) Create project
    my $project_id = create_project($project,$_dbh); 

    ## 2) Use project_id to create versions
    my $versions = get_versions(\@function_files);
    my $version_data = {}; 
    foreach my $version (@{$versions})
    {
        my $version_id = create_version($project_id,$_dbh,$version);  
        #print "VERSION: $version_id and $version \n";
        $version_data{$version} = $version_id;
    }

    ## 3) Use version_id to create statistics
    create_statistics(\%version_data,$_dbh,$_stats_file);

    ## 4) Use version_id to create files
    ## 5) Use file_id to create components   
    my $current_version;
    foreach my $_file (@function_files)
    {
        ## get the version id for the file contents
        foreach my $_file_key (keys %version_data)
        {
            if ($_file =~ /$_file_key/)
            {
                $current_version = $version_data{$_file_key}; 
                break;
            }
        }

        ## go through the file and insert files for the version
        open(FUNCTIONS_HANDLE,$_file);    
        my @function_file_line_data;
        my @file_info;
        my $file_id;
        my $component_name;
        my $component_type;
        my $component_line_number;
        my $component_value;
        while(<FUNCTIONS_HANDLE>)
        {
            @function_file_line_data = split(" ", $_,5); 
            @file_info = split("\/",$function_file_line_data[3],3);

            #if ($file_info[2] eq " ")
            #{ exit };
            #print "FILE: $file_info[2] \n";
            
            ## Skip malformed lines
            next if $file_info[2] eq "";

            $file_id = create_file($current_version,$_dbh,$file_info[2]);
            $component_name = $function_file_line_data[0]; 
            $component_type = $function_file_line_data[1]; 
            $component_line_number = $function_file_line_data[2]; 
            #print "LINE #: $component_line_number \n";
            $component_value = $function_file_line_data[4]; 
            create_component($file_id,$_dbh,$component_name,$component_type,
                             $component_line_number,$component_value);  
        }

        close(FUNCTIONS_HANDLE);
 
    }

    ## Close datasource
    $_dbh->disconnect();
}

## Run the program
&main($ARGV[0],$_DB_HANDLE,$_BASE,$_STATS_FILE);
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.