#! /usr/bin/perl
# Process export requests.
# 
#  Copyright (C) 2005-2006 Yves Perrin <yves.perrin--cern.ch>
#                          BBN Technologies Corp
#  Copyright (C) 2018 Ineiev
#
# This file is part of Savane.
# 
# Savane is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
# 
# Savane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
                                                                                
use strict;
use Savane;
use Savane::Mail;
use Getopt::Long;
use Term::ANSIColor qw(:constants);
use POSIX qw(strftime);
use Time::Local;
use Date::Calc qw(Add_Delta_YMD);

use XML::Writer;
use IO::File;

# Import
our $sys_name;
our $sys_https_host;
our $sys_default_domain;
our $sys_url_topdir;

# Configure
my $default_logfile = "/var/log/sv_export.log";
my $script = "sv_export";
my $logfile = $default_logfile;

my $getopt;
my $help;
my $xml_path;
my $debug;

eval {
    $getopt = GetOptions("help" => \$help,
                         "xml-path=s" => \$xml_path,
                         "log-file=s" => \$logfile,
                         "debug" => \$debug);
};
 
if($help) {
    print STDERR <<EOF;
usage: $0
 
   Search the database 'trackers_export' table for pending
   requests and for each of these, extract the relevant data,
   produce the corresponding XML representation and
   write it together with the appropriate XML schema into a file
   the path of which being specified by the xml-path argument.
 
        --help                  print this help
        --xml-path=/            path of the generated xml file
        --log-file=$default_logfile
                                path for log file
 
Author: yves.perrin\@cern.ch, yeupou\@gnu.org

Copyright (C) 2005-2006 Yves Perrin <yves.perrin--cern.ch>
                        BBN Technologies Corp
Copyright (C) 2018 Ineiev

License AGPLv3+: GNU AGPL version 3 or (at your option)
any later version <https://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.

EOF
  exit(0);
}

unless (-w $logfile) {
  die "Log file $logfile is not writable, exiting";
}

#Starts only if the xml-path exists.
unless (-d $xml_path) {
  print LOG "[$script] $xml_path does not exists or is not a directory\n";
  die "$xml_path does not exists or is not a directory, exiting",
}

#Locks: instances should not run concurrently, so we add a lock.
AcquireReplicationLock();

# Log: Starting logging
open (LOG, ">>$logfile");
print LOG strftime "[$script] %c - starting\n", localtime;

# get the current date and time
my $now = time();

# get the pending export requests
# trackers_export table fields:
#    export_id
#    artifact          = tracker
#    unix_group_name   = savane project name
#    user_name         = login name of requestor
#    sql               = sql to retrieve the items ids (only)
#    status            = P  (pending, to be performed)
#                        D  (done, removed by the user with the interface)
#    date              = unix timestamp.
#                        generate xml if current time > date
#    frequency         = if these are set, status is not set to D when
#                        the job has been done; it remains equal to Q
#                        but the date is updated accordingly
#             _day     = values from 1 (monday) to 7 (sunday) + 8 (everyday)
#             _hour    = values from 0 (midnight) to 23
#
# Note: field labels are used in lower-case with blank space replaced by _
# In theory, if a project set two fields labels to exactly match, broken xml
# will be produced. But that problem could be easily solved by renaming the
# incriminated fields.

my $export_table = 'trackers_export';
my $fields = 'export_id, task_id, artifact, unix_group_name, user_name, `sql`, status, date, frequency_day, frequency_hour';
my $criteria = "date < $now AND status='P'";

my %jobs = GetDBHash($export_table, $criteria, $fields);
while (my ($key, $value) = each(%jobs)) {
    my ($req_id, $req_task_id, $tracker, $req_group, $req_user, $req_ids_sql, $req_status, 
	$req_date, $req_frequency_day, $req_frequency_hour) = (@{$value});

    my @item_ids = GetDBAsIs($req_ids_sql);
    my $group_id = GetGroupSettings($req_group, 'group_id');
    my $group_path = $xml_path.'/'.$req_group;

    if (!(-e $group_path)) {
	mkdir($group_path,0755);
    }
    my $user_path = $group_path.'/'.$req_user;
    
    if (!(-e $user_path)) {
	mkdir($user_path,0755);
    }
    my $xmlfile = $user_path.'/'.$req_id.'.xml'; 
    my $xmlschema = $user_path.'/'.$req_id.'.xsd'; 

    my %tr_fields_project;
    my %tr_fields_values;

    # ----------------------
    # to be cleaned
    my ($pos, $tables_val, $fields_val, $criteria_val, $key);
    my ($sel_fields, $criteria_base, $s_no_item, $fn, $fv, $unix_history_date);
    my ($history_date, $sel_fields_hist, $tables_hist, $criteria_hist);
    my ($sel_fields_dep, $tables_dep, $criteria_dep);
    my (%this_item, $thisid, $fd, $id, $item, $label, $d, $f);
    # ----------------------

    # For ALL fields get DEFAULT usage
    my $tables = $tracker.'_field, '.$tracker.'_field_usage';
    my $fields = $tracker.'_field.field_name, '.$tracker.'_field.label, '.$tracker.'_field.bug_field_id, '.$tracker.'_field.display_type, '.$tracker.'_field_usage.use_it';
    my $criteria = $tracker.'_field_usage.group_id=100 AND  '.
	$tracker.'_field.bug_field_id='.
	$tracker.'_field_usage.bug_field_id ';
    
    my %usage = GetDBHash($tables, $criteria, $fields);
    while (my ($key, $value) = each(%usage)) {
	my ($fname, $flabel, $fid, $fdisptype, $fuse) = (@{$value});
	$tr_fields_project{$fname} = ();
	$tr_fields_project{$fname}{'label'} = $flabel;
	$tr_fields_project{$fname}{'id'} = $fid;
	$tr_fields_project{$fname}{'display_type'} = $fdisptype;
	$tr_fields_project{$fname}{'use'} = $fuse;
    }
    

    # Update %tr_fields_project according to project specific usage entries

    $tables = $tracker.'_field, '.$tracker.'_field_usage';
    $fields = $tracker.'_field.field_name, '.$tracker.'_field_usage.use_it';
    $criteria = 'group_id='.$group_id.' AND  '.
	$tracker.'_field.bug_field_id='.
	$tracker.'_field_usage.bug_field_id ';
    
    my %usage = GetDBHash($tables, $criteria, $fields);
    while (my ($key, $value) = each(%usage)) {
	my ($fname, $fuse) = (@{$value});
	$tr_fields_project{$fname}{'use'} = $fuse;
    }

    # Get the field types and keep only the 'USED' fields                                                                               
    my %tr_fields_arr;
    foreach my $line (GetDBDescribe($tracker)) {
	chomp($line);
	my ($tr_field_name,$tr_field_type) = split(",", $line);
	$pos = -1;
	$pos = index($tr_field_type, 'int(');
	if ($pos != -1) {
	    $tr_field_type = 'integer';
	} else {
	    $pos = index($tr_field_type, 'text');;
	    if ($pos != -1) {
		$tr_field_type = 'string';
	    } else {
		$pos = index($tr_field_type, 'char(');;
		if ($pos != -1) {
		    $tr_field_type = 'string';
		} else {
		    $pos = index($tr_field_type, 'float(');;
		    if ($pos != -1) {
			$tr_field_type = 'decimal';
		    }
		}
	    }
	}
	if ($tr_fields_project{$tr_field_name}{'use'} == 1) {
	    $tr_fields_arr{$tr_field_name} = $tr_field_type;
	    if ($tr_fields_project{$tr_field_name}{'display_type'} eq 'SB') {
		# get the field values

		# Look for project specific values first
		$tables_val = $tracker."_field_value ";
		$fields_val = "value_id, value ";
		$criteria_val = "group_id=$group_id ".
		    "AND bug_field_id=$tr_fields_project{$tr_field_name}{'id'} ".
		    "AND  status IN ('A','P') ";

		
		my %values = GetDBHash($tables_val, $criteria_val, $fields_val);
		while (my ($key, $value) = each(%values)) {
		    my ($vid, $val) = (@{$value});
		    $tr_fields_values{$tr_fields_project{$tr_field_name}{'id'}}{$vid} = $val;
		}

		# If no specific value for this group then look for default values
		if (!exists($tr_fields_values{$tr_fields_project{$tr_field_name}{'id'}})) {
		    $tables_val = $tracker."_field_value ";
		    $fields_val = "value_id, value ";
		    $criteria_val = "group_id=100 ".
			"AND bug_field_id=$tr_fields_project{$tr_field_name}{'id'} ".
			"AND  status IN ('A','P') ";
		    
		    
		    %values = GetDBHash($tables_val, $criteria_val, $fields_val);
		    while (my ($key, $value) = each(%values)) {
			my ($vid, $val) = (@{$value});
			
			$tr_fields_values{$tr_fields_project{$tr_field_name}{'id'}}{$vid} = $val;
		    }
		}
	    }
	}
    }
    

    
    # ---------------------------------------
    # generate the xml schema
    #
    # FIXME: how useful the schema is, actually?
    open (XMLSCHEMA, ">$xmlschema");
    print XMLSCHEMA '<?xml version="1.0" encoding="utf-8"?>'."\n".
	'<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">'."\n";
    print XMLSCHEMA '<xs:element name="savaneexport">'."\n";                            
    print XMLSCHEMA '  <xs:complexType>'."\n";
    print XMLSCHEMA '    <xs:element name="item">'."\n";
    print XMLSCHEMA '    <xs:complexType>'."\n";
    print XMLSCHEMA '      <xs:element name="tracker" type="string" />'."\n";
    foreach $key (keys(%tr_fields_arr)) {
	my $element = lc($tr_fields_project{$key}{'label'});
	$element =~ s/\s/_/g;
	$element =~ s/\//_/g;
	$element =~ s/\#//g;
	print XMLSCHEMA '      <xs:element name="'.$element.'" type="'.$tr_fields_arr{$key}.'" />'."\n";
    }

    print XMLSCHEMA '      <xs:element name="depends_on" minOccurs="0">'."\n";
    print XMLSCHEMA '        <xs:complexType>'."\n";
    print XMLSCHEMA '          <xs:element name="item_id" type="integer" />'."\n";
    print XMLSCHEMA '          <xs:element name="tracker" type="string" />'."\n";
    print XMLSCHEMA '        </xs:complexType>'."\n";
    print XMLSCHEMA '      </xs:element>'."\n";
    print XMLSCHEMA '      <xs:element name="history" minOccurs="0" maxOccurs="1">'."\n";
    print XMLSCHEMA '        <xs:complexType>'."\n";
    print XMLSCHEMA '          <xs:element name="event" minOccurs="1">'."\n";
    print XMLSCHEMA '            <xs:complexType>'."\n";
    print XMLSCHEMA '              <xs:element name="date" type="date" />'."\n";
    print XMLSCHEMA '              <xs:element name="field" minOccurs="1">'."\n";
    print XMLSCHEMA '                <xs:complexType>'."\n";
    print XMLSCHEMA '                  <xs:element name="field_name" type="string" />'."\n";
    print XMLSCHEMA '                  <xs:element name="old_value" type="string" />'."\n"; 
    print XMLSCHEMA '                  <xs:element name="new_value" type="string" />'."\n";
    print XMLSCHEMA '                  <xs:element name="modified_by" type="string" />'."\n";
    print XMLSCHEMA '                </xs:complexType>'."\n";
    print XMLSCHEMA '              </xs:element>'."\n";
    print XMLSCHEMA '            </xs:complexType>'."\n";
    print XMLSCHEMA '          </xs:element>'."\n";
    print XMLSCHEMA '      </xs:complexType>'."\n";
    print XMLSCHEMA '      </xs:element>'."\n";
    print XMLSCHEMA '    </xs:complexType>'."\n";
    print XMLSCHEMA '    </xs:element>'."\n";
    print XMLSCHEMA '  </xs:complexType>'."\n";
    print XMLSCHEMA '</xs:element>'."\n";
    print XMLSCHEMA '</xs:schema>'."\n";
    close(XMLSCHEMA);

    # ------------- build the sql command to get all items data --------------
    
    $sel_fields = 'DISTINCT ';
    $tables = $tracker;
    $criteria_base = '';

    $s_no_item = 1;
    
    while (($fn, $fv) = each(%tr_fields_arr)) {
	if (!$s_no_item) {
	    $sel_fields .= ",";
	}
	if (($fn eq 'assigned_to') || ($fn eq 'submitted_by')) {
	    # user names requires some special processing to display the
	    # username instead of the user_id
	    $sel_fields .= " user_$fn.user_name AS $fn";
	    $tables .= ", user user_$fn";
	    $criteria_base .= " AND user_$fn.user_id=".$tracker.".$fn ";
	} else {
	    # otherwise just select this column as is
	    $sel_fields .= " ".$tracker.".$fn";
	}
	$s_no_item = 0;
    }

    # -------------------- Generate the XML data ----------------------

    print LOG strftime "[$script] %c - start writing $xmlfile\n", localtime if $debug;
    my $xml = new IO::File(">$xmlfile");
    # UNSAFE has been turned on because on big installation, some project
    # were able to put content that XML::Writer is not able to convert 
    # properly. We have to think about the issue.
    # In the meantime, our script has been verified to provide valid XML 
    # in the structure and in the content with normal data. So turning the
    # safety checks off is probably the best thing to do for now.
    my $writer = new XML::Writer(OUTPUT => $xml,
				 DATA_MODE => 1,
				 DATA_INDENT => 2,
				 UNSAFE => 1);
    $writer->xmlDecl("UTF-8");
    $writer->comment(strftime "Generated on %c", localtime);
    $writer->comment("A XML Schema is available at the same url, with the suffix .xsd instead of .xml");
    $writer->startTag("savaneexport");

    my @fn = split(",", $sel_fields);
    foreach $id (@item_ids) {

	# --------------------- Get the items history ------------------------

	$sel_fields_hist = 'bug_id, date, field_name, old_value, new_value, user.user_name ';
	$tables_hist = $tracker.'_history, user ';
	$criteria_hist = 'bug_id='.$id.' AND user.user_id='.$tracker.'_history.mod_by ORDER BY date';
        my %history_hash = {}; #hash elements will be hash of array of hashes !
	
	my %hist_row = GetDBHash($tables_hist,$criteria_hist,$sel_fields_hist);
	while (my ($key, $value) = each(%hist_row)) {
	  my ($h_bug_id, $h_date, $h_field, $h_f_old, $h_f_new, $h_f_modby) =
             (@{$value});
	    
	  if (!exists($history_hash{$h_bug_id})) {
	    $history_hash{$h_bug_id} = {}; # possibly many events for same id
	  }
	  if (!exists($history_hash{$h_bug_id}{$h_date})) {
	    $history_hash{$h_bug_id}{$h_date} = (); # possibly many fields for same date
            $history_hash{$h_bug_id}{$h_date}[0] = {
                                      field => $h_field,
                                      old_value  => $h_f_old,
                                      new_value  => $h_f_new,
                                      mod_by     => $h_f_modby };
	  } else {
            my %this_hist_date_event = {};
            $this_hist_date_event{'field'} = $h_field;
            $this_hist_date_event{'old_value'}  = $h_f_old;
            $this_hist_date_event{'new_value'}  = $h_f_new;
            $this_hist_date_event{'mod_by'}     = $h_f_modby;
          
            # push ($history_hash{$h_bug_id}{$h_date}, %this_hist_date_event);
            my $hist_date_index = scalar @{$history_hash{$h_bug_id}{$h_date}};
            # scalar returns length, so last element +1 ... no need to pre-inc
            %{$history_hash{$h_bug_id}{$h_date}[$hist_date_index]} = 
                 %this_hist_date_event;
          }
	}

        # --------------------- Get the items dependencies -------------------
                                                                                
        $sel_fields_dep = 'is_dependent_on_item_id, is_dependent_on_item_id_artifact ';
        $tables_dep = $tracker.'_dependencies ';
        $criteria_dep = 'item_id='.$id;
        my %dep_hash = {}; #hash elements will be array of hashes !!!
                                                                                
        my %dep_row = GetDBHash($tables_dep,$criteria_dep,$sel_fields_dep);
        my $i = 0;
        while (my ($key, $value) = each(%dep_row)) {
          my %this_dep = {};
          ($this_dep{'item_id'}, $this_dep{'tracker'}) = (@{$value});
          if (!exists($dep_hash{$id})) {
            $dep_hash{$id} = ();
          }
          %{$dep_hash{$id}[$i]} = %this_dep;
          $i++;
        }
                                                                                
	# ---------------------- Get the items data --------------------------
	
	$criteria = $tracker.'.bug_id='.$id.$criteria_base;  

	my %item_row = GetDBHash($tables,$criteria,$sel_fields);
	while (my ($key, $value) = each(%item_row)) {
	    my @f = (@{$value});

	    $writer->startTag("item");
	    $writer->startTag("tracker");
	    $writer->characters($tracker);
	    $writer->endTag("tracker");;

	    my $k = 0;
	    foreach my $fn (keys (%tr_fields_arr)) {
		$label = lc($tr_fields_project{$fn}{'label'});
		$label =~ s/\s/_/g;
		$label =~ s/\//_/g;
		$label =~ s/\#//g;
		if ($tr_fields_project{$fn}{'display_type'} eq 'SB') {
		    $writer->startTag($label);
		    if (($fn eq 'submitted_by') || ($fn eq 'assigned_to')) {
			$writer->characters($f[$k]);
		    } else {
			$writer->characters($tr_fields_values{$tr_fields_project{$fn}{'id'}}{$f[$k]});
		    }
		    $writer->endTag($label);
		} else {
		    $writer->startTag($label);
		    $writer->characters($f[$k]);
		    $writer->endTag($label);
		}
		$k++;
	    }

            if (exists($dep_hash{$id})) {
                my $nb_of_dep = scalar @{$dep_hash{$id}};
                my $i = 0;
                while ($i < $nb_of_dep) {
                  $writer->startTag("depends_on");
                    $writer->startTag("item_id");
#                   my $itid = ${$dep_hash{$id}[$i]}{'item_id'};
#                   $writer->characters($itid);
                    $writer->characters(${$dep_hash{$id}[$i]}{'item_id'});
                    $writer->endTag("item_id");
                    $writer->startTag("tracker");
                    $writer->characters(${$dep_hash{$id}[$i]}{'tracker'});
                    $writer->endTag("tracker");
                  $writer->endTag("depends_on");
                  $i++;
                }
            }

	    if (exists($history_hash{$id})) {
		$writer->startTag("history");

		for $d ( sort keys %{$history_hash{$id}}) {
		    $writer->startTag("event");
		    $writer->startTag("date");
		    $writer->characters($d);
		    $writer->endTag("date");

                    my $i = 0;
                    my $nb_of_fields = scalar @{$history_hash{$id}{$d}};
                    while ($i < $nb_of_fields) {
                        my $h_f = $history_hash{$id}{$d}[$i]{'field'};
			$writer->startTag("field");

                        $label = $tr_fields_project{$h_f}{'label'};

			$label =~ s/\s/_/g;

			$writer->startTag("field_name");
			$writer->characters($label);
			$writer->endTag("field_name");
			
           $writer->startTag("old_value");
           if ($tr_fields_project{$h_f}{'display_type'} eq 'SB') {
             if (($h_f eq 'submitted_by') || ($h_f eq 'assigned_to')) {
               $writer->characters($history_hash{$id}{$d}[$i]{'old_value'});
             } else {
               $writer->characters($tr_fields_values{$tr_fields_project{$h_f}{'id'}}{$history_hash{$id}{$d}[$i]{'old_value'}});
             }
           } else {
             $writer->characters($history_hash{$id}{$d}[$i]{'old_value'});
           }
           $writer->endTag("old_value");

           $writer->startTag("new_value");
           if ($tr_fields_project{$h_f}{'display_type'} eq 'SB') {
             if (($h_f eq 'submitted_by') || ($h_f eq 'assigned_to')) {
               $writer->characters($history_hash{$id}{$d}[$i]{'new_value'});
             } else {
               $writer->characters($tr_fields_values{$tr_fields_project{$h_f}{'id'}}{$history_hash{$id}{$d}[$i]{'new_value'}});
             }
           } else {
             $writer->characters($history_hash{$id}{$d}[$i]{'new_value'});      
           }
           $writer->endTag("new_value");

			$writer->startTag("modified_by");
			$writer->characters($history_hash{$id}{$d}[$i]{'mod_by'});
			$writer->endTag("modified_by");

			$writer->endTag("field");
                        $i++;
		    }
		    $writer->endTag("event");
		}
		$writer->endTag("history");
	    }
	    $writer->endTag("item");
	}
    } 
    $writer->endTag("savaneexport");
    $xml->close();
    
    # --------------------- Notice the change ------------------------
    # should:
    #    - update the status
    #    - update the timestamp in case of a frequent job
    #    - post a comment on the relevant task
    
    print LOG strftime "[$script] %c - export #$req_id processed\n", localtime;
    
    if (!$req_frequency_day && !$req_frequency_hour) {

	# One-time export request: update the status
	SetDBSettings($export_table, "export_id='$req_id'", "status='D'");

    } else {

        # insert completion comment in export task
        my $task_comment = "... Requested items have been exported";
        InsertDB("task_history",
                 "bug_id, field_name, old_value, mod_by, date",
                 "'$req_task_id', 'details', '$task_comment', '100', '$now'");
                                                                                
	# Otherwise, update the timestamp
	# Assume that the script is running the correct day, so we can easily
	# just add 7 days to the current day
	my ($year, $month, $day) = split(",", `date +%Y,%m,%d`);
        if ($req_frequency_day < 8) {
	  ($year,$month,$day) = Add_Delta_YMD($year,$month,$day, 0,0,7);
        } else {
          ($year,$month,$day) = Add_Delta_YMD($year,$month,$day, 0,0,1);
        }
	my $timestamp = timelocal("0","0",$req_frequency_hour,$day,($month-1),($year-1900));
	SetDBSettings($export_table, "export_id='$req_id'", "date='$timestamp'"); 

    }
       
	
}

# Final exit
print LOG strftime "[$script] %c - work finished\n", localtime;
print LOG "[$script] ------------------------------------------------------\n";


# EOF
