#!/usr/bin/perl 
use strict;
use warnings;
use lib '/usr/local/lib/mpmasql/modules';

use Storable qw(dclone);
use EmpiricalDF;
use Carp;
use POSIX;
use List::Util qw(max);
use Scalar::Util qw(looks_like_number);
use Data::Dumper qw(Dumper);
use Term::ReadKey;

#use Math::Random::MT::Auto qw/rand srand/ ???

use CtUtilities qw(cartesian is_in_list fisher_yates_shuffle determine_class_by_upperbound);
use IndexedTable;
use MpmasQLConfigReader;
use MpmasQLVariableProcessing;
use MpmasQLFunctionLanguage;
use MasMapAgent;
use MasMapRegion;
use EnhancedGrid;
use SimpleObject;
use SimpleObjectRegistry;

use MpmasDistSharedFunctions qw(prepare_icdf predraw_from_uniform draw_value_from_cdf do_assignment do_hhmember_assignment do_post_assignment evaluate_constraints divide_constraints set_verbosity);
use MpmasDistMainDistTypes;
use MpmasDistLatynskiy;

use constant INFTY => 10**31;

my %known_disfunctions = (
#Core:
	assign_by_cdf => \&MpmasDistMainDistTypes::assign_by_cdf,
	distribute_observation_list => \&MpmasDistMainDistTypes::distribute_observation_list,
	fit_into_subarea => \&MpmasDistMainDistTypes::fit_into_subarea,
	fixed  => \&MpmasDistMainDistTypes::fixed,
	fixed_repeated  => \&MpmasDistMainDistTypes::fixed_repeated,
	make_secondary_maps_for_mpmas  => \&MpmasDistMainDistTypes::make_secondary_maps_for_mpmas,
	maps_artificial  => \&MpmasDistMainDistTypes::maps_artificial,
	maps_assign_farmstead  => \&MpmasDistMainDistTypes::maps_assign_farmstead,
	maps_assign_plots  => \&MpmasDistMainDistTypes::maps_assign_plots,
	match  => \&MpmasDistMainDistTypes::match,
	match_cdf  => \&MpmasDistMainDistTypes::match_cdf,
	none  => \&MpmasDistMainDistTypes::none ,
#Evgeny:	
	fit_uniform_to_total => \&MpmasDistLatynskiy::fit_uniform_to_total,
	fit_cdf_to_total => \&MpmasDistLatynskiy::fit_cdf_to_total
);
#$Carp::Verbose  = 1;
##Version:
my $thisversion = "v2.02";
my $lastchange = "13-Mar-2019";
my $starttime = time;

my $longwelcome = <<WC;


####################################################################
####			mpmasdist $thisversion		####
####	Perl script to distribute endowments to MP-MAS agents	####
####								####
####	(c)	Christian Troost, Univ. Hohenheim, 2011-2013	####
####	Version: $thisversion		$lastchange		####
####################################################################
WC

my $shortwelcome = <<WC2;
mpmasdist $thisversion		
(c)	Christian Troost, Universität Hohenheim, 2011-2013
WC2

my $ininame;
my %setstrings;
while (@ARGV) {
	my $element = shift @ARGV;
	if  ($element =~ /^-v$/) {
		print $longwelcome;
		exit 0;
	}
	elsif (not $element =~ /^-/) {
		# inifile name
		$ininame = $element;
		print "\nControl file:\t\t". $ininame."\n";  
	}
	elsif ($element =~ /^--set/ ) {
		if (not $ARGV[0] =~ /^-/ ) {
		    my $setstring = shift @ARGV;
		    my ($var, $val) = split /=/,$setstring,2;
		    $var =~ s/^\s*//;
		    $val =~ s/^\s*//;
		    $var =~ s/\s*$//;
		    $val =~ s/\s*$//;

		    $setstrings{$var} = $val;
		}
	}
}

#Give error message if no input file was specified
print STDERR "You forgot to specify a control file!\n" and exit(1) unless $ininame;

######## Read Ini-File
my ($inihash, $setup, $tables, $discollect, $ppinfo,$varrank,  $maps, $stepsclude) = MpmasQLConfigReader::ReadDst($ininame, 0);
######## Connect to Db
my $dbh;
ReadMode(0);
if (defined  $inihash->{DBTYPE} and defined $inihash->{DB}) {
	if (not $inihash->{DBPWD}) {
		print "Password: ";
		ReadMode 2;	
		 $inihash->{DBPWD} = ReadLine (0);
		print "\n";
		ReadMode 1;
	}
	chomp( $inihash->{DBPWD});
	$dbh = DBI->connect("dbi:".$inihash->{DBTYPE}.":". $inihash->{DB}, $inihash->{DBUSER}, $inihash->{DBPWD}, { RaiseError => 1, AutoCommit => 1 }) or die $!;
	print "Successfully connected to " . $inihash->{DB} . " as ". $inihash->{DBUSER} . "\n"; 
	$dbh->do("SET NAMES utf8");
}


#interprete variables
MpmasQLFunctionLanguage->set_globals(0,$dbh,{},{},0);
for my $i (sort {$varrank->{$a} <=> $varrank->{$b} } keys %$setup) {
  if (exists $setstrings{$i}) {
	$setup->{$i} = MpmasQLFunctionLanguage->convert($setstrings{$i}, {}) ;
  }
  else{
	$setup->{$i} = MpmasQLFunctionLanguage->convert($setup->{$i}, {});
  }
  MpmasQLFunctionLanguage->set_globals( 0, $dbh, $setup,"",0);
}
#mkpaths
if (exists $stepsclude->{MKPATHS}) {
  for my $p ( MpmasQLFunctionLanguage->convert($stepsclude->{MKPATHS}, {})) {
	system("mkdir -p ". $p);
  }
}

#load tables
my $tableregistry= SimpleObjectRegistry->create();
for my $l (keys %{$tables}){
    $tableregistry->add_objects($tables->{$l}->transform($dbh));
}
MpmasQLFunctionLanguage->set_globals( 0, $dbh, $setup,$tableregistry);

#load maps
my $mapregistry = SimpleObjectRegistry->create();
for my $m (keys %$maps) {
	
	my $mapfile = MpmasQLFunctionLanguage->convert($maps->{$m},{});
	my $newmap = EnhancedGrid->create_from_ascii($mapfile);
	$newmap->set_id($m);
	$mapregistry->add_objects($newmap);
}
MpmasQLFunctionLanguage->set_mapregistry($mapregistry);

#setup list of steps to process
my @steplist;
if (exists $stepsclude->{INCLUDE}) {
    @steplist = MpmasQLFunctionLanguage->convert($stepsclude->{INCLUDE},{});
}
else {
      @steplist = sort {$a <=> $b} keys %$discollect;
}
if (exists $stepsclude->{EXCLUDE}) {
  my @exclude = MpmasQLFunctionLanguage->convert($stepsclude->{EXCLUDE},{});
  my %testhash = map { $_ => 1} @exclude;
  my @newlist = map { exists $testhash{$_} ? () : $_  } @steplist;
  @steplist = @newlist;
}
print "Steps to process: ".join (",", @steplist)."\n";



#declare 
my $total_agent_registry = SimpleObjectRegistry->create();
my $notassigned = SimpleObjectRegistry->create();
my $subarearegistry = SimpleObjectRegistry->create();




for my $d (@steplist) {
	
	my $disinfo = $discollect->{$d}; 
	my $seed;
	if (exists $disinfo->{SEED}) {
		$seed = MpmasQLFunctionLanguage->convert($disinfo->{SEED},{});
		srand($seed) if (defined $seed); 
	}
	print "STEP $d: ";
	print $disinfo->{TYPE};
	print " SEED: ". $seed if (defined $disinfo->{SEED});
	my  ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
	$mon += 1;
	$year += 1900;
	print "\tTIME: $year $mon $mday , $hour:$min:$sec\n";
	if (exists $disinfo->{DESCRIPTION}) {
		my $desc = MpmasQLFunctionLanguage->convert($disinfo->{DESCRIPTION},{});
		print $desc."\n" if (defined $desc);
	}
	my $verbose = $disinfo->{VERBOSE}||0;
	my $constraints = divide_constraints($disinfo);
	
	
	my %clusters;
	my $cc = 0;
	my @combined = ([(0)]);
	my @clulist = ("grp");
	if (exists $disinfo->{LOOPS}->{lines} and @{$disinfo->{LOOPS}->{lines}} > 0) {
		undef @clulist;
		for my $i (0..$#{$disinfo->{LOOPS}->{lines} }) {
			my ($field, $content) = split /\s*#=\s*/, $disinfo->{LOOPS}->{lines}->[$i];
			my $key = MpmasQLFunctionLanguage->convert($field,{});
			push @clulist, $key;
			$clusters{$key} = [(MpmasQLFunctionLanguage->convert($content,{}) )];	
		}
		
		
		undef @combined;
		
		if (scalar @clulist == 1) {
			@combined = map { [ $_ ] }  @{$clusters{$clulist[0]} };
		}
		else {
		  my @temp = map { $clusters{$_} } @clulist;
		  foreach (@clulist) {
			@combined = cartesian (@temp );
		  }
		}
	}
	my $agentregistry = SimpleObjectRegistry->create(); #declared outside foreach, so it is visible in continue, make sure it is always cleared!
	
	CLULOOP: foreach my $clu (@combined) {
		my $locvarhash = {}; 
		foreach my $i (0..$#clulist) {
			
			$locvarhash->{$clulist[$i]} = $clu->[$i];
		}
		print "\e[K \r\tLOOP: @$clu -> \n" if (scalar @combined > 1);
		#map {print join("\t", $_, $locvarhash->{$_})."\n" } keys %$locvarhash;

		
		if (not defined $disinfo->{AGENTS}->{list} or $disinfo->{AGENTS}->{list} =~ /^\s*$/ or $disinfo->{AGENTS}->{list} eq "AGENTS" ) {
			$agentregistry = dclone($total_agent_registry);	
			$total_agent_registry->clear();
		}
		elsif ($disinfo->{AGENTS}->{list} =~ /^\s*RESTORE\s+(.*)$/)  {
			my $fn = MpmasQLFunctionLanguage->convert($1,$locvarhash);
			$total_agent_registry->clear();
			$agentregistry = SimpleObjectRegistry->recreate($fn);
			print "AGENTS restored from $fn\n";
		}	
		else {
			my @agentlist = MpmasQLFunctionLanguage->convert($disinfo->{AGENTS}->{list},$locvarhash);
			print "Creating ". scalar @agentlist. " new agents\n" if $verbose;
			$agentregistry = SimpleObjectRegistry->create();
			next CLULOOP if ( not defined $agentlist[0]);
			map { my $aid = $_; my $newagent = MasMapAgent->create(id => $aid,num_assigned => 0, num_members => 0, assigned => SimpleObjectRegistry->create(), memberlist => SimpleObjectRegistry->create() ); $agentregistry->add_objects($newagent); } @agentlist
		}
		for my $curAgent ($agentregistry->object_list) {
			my %locvars1  = %$locvarhash;
			$locvars1{agent_id} = $curAgent->id;
			$locvars1{agent} = $curAgent;
			for my $i (0..$#{$disinfo->{AGENTS}->{lines}}) {
				my ($field, $content) = split /\s*#=\s*/, $disinfo->{AGENTS}->{lines}->[$i];
				my ($keys, $fors) = MpmasQLFunctionLanguage->translate_and_interprete($field,\%locvars1);
				for my $j (0..$#{$keys}) {		
					my %locvars2  = %locvars1;
					$locvars2{attribute} = $fors->[$j];
					$curAgent->set_attribute($keys->[$j], MpmasQLFunctionLanguage->convert($content,\%locvars2) ||0 );
				}
			}
		}
		my $observationregistry = SimpleObjectRegistry->create();
		if (not defined $disinfo->{OBSERVATIONS}->{list} or $disinfo->{OBSERVATIONS}->{list} =~ /^\s*$/) {
			if ( is_in_list($disinfo->{TYPE}, [( qw(assign_by_cdf distribute_observation_list ) )]) ) {
					print STDERR "Error: You did not provide an OBSERVATIONS list for STEP $d\n";
					exit(1);
			}
		}
		elsif ($disinfo->{OBSERVATIONS}->{list} eq "NOTASSIGNED" ) {
			next unless ($notassigned->counter > 0);
			$observationregistry = dclone ($notassigned);
			$notassigned->clear ;
		}
		elsif ($disinfo->{OBSERVATIONS}->{list} =~ /^\s*RESTORE\s+(.*)$/)  {
			my $fn = MpmasQLFunctionLanguage->convert($1,$locvarhash);
			$notassigned->clear();
			$observationregistry = SimpleObjectRegistry->recreate($fn);
			print "OBSERVATIONS restored from $fn\n";
		}	
	
		else {
			my @observationlist = MpmasQLFunctionLanguage->convert($disinfo->{OBSERVATIONS}->{list},$locvarhash);
			next unless (defined $observationlist[0]);
			map { my $eid = $_; my $newobservation = SimpleObject->create(id => $eid); $observationregistry->add_objects($newobservation); } @observationlist
		}

		for my $curEndow ($observationregistry->object_list()) {
			my %locvars1  = %$locvarhash;
			#print $curEndow->get_attribute("id");
			$locvars1{observation_id} = $curEndow->get_attribute("id");
			$locvars1{observation} = $curEndow;
			for my $i (0..$#{$disinfo->{OBSERVATIONS}->{lines}}) {
				my ($field, $content) = split /\s*#=\s*/, $disinfo->{OBSERVATIONS}->{lines}->[$i];
				$|=1;
				my ($keys, $fors) = MpmasQLFunctionLanguage->translate_and_interprete($field,\%locvars1);
				for my $j (0..$#{$keys}) {		
					my %locvars2  = %locvars1;		
					$locvars2{attribute} = $fors->[$j];
					$curEndow->set_attribute($keys->[$j], MpmasQLFunctionLanguage->convert($content,\%locvars2) || 0 );
				}
			}
			#defaults:
			$curEndow->set_attribute("DEL_priority", INFTY) unless (defined $curEndow->get_attribute("DEL_priority"));
			$curEndow->set_attribute("CDF_priority", INFTY) unless (defined $curEndow->get_attribute("CDF_priority"));
			#$newagent->dump_me_to();
			if (defined $curEndow->get_attribute("OBS_multiply") and $curEndow->get_attribute("OBS_multiply") > 1) {
			      $observationregistry->delete_objects($curEndow);
			      foreach (1..$curEndow->get_attribute("OBS_multiply")) {
				    my $o = $curEndow->create_copy();
				    $o->set_attribute("id", $curEndow->get_attribute("id")* -100000 + $_);
				    $observationregistry->add_objects($o);
			      }
			}
		
		}
		if (exists $known_disfunctions{ $disinfo->{TYPE} } ) {
			&{ $known_disfunctions{ $disinfo->{TYPE} } }($disinfo, $agentregistry, $observationregistry, $notassigned, $mapregistry, $subarearegistry, $locvarhash, $constraints, $verbose, $d, $clu) ;
		}
		else {
			print STDERR "Unknown distribution type $disinfo->{TYPE} in STEP $d\n";
			exit(1);
		}
	} continue {
		$total_agent_registry->add_objects($agentregistry->object_list());
		$agentregistry->clear();
		print "Number of agents after STEP $d, GROUP @$clu: ". $total_agent_registry->counter()."\n" if $verbose;
		
		
	}
	for my $curAgent ($total_agent_registry->object_list) {
		my %locvars1;
		$locvars1{agent_id} = $curAgent->id;
		$locvars1{agent} = $curAgent;
		if (exists $disinfo->{DROP_IF}) {
			my $drop = MpmasQLFunctionLanguage->convert( $disinfo->{DROP_IF},\%locvars1);
			if ($drop) {
				$total_agent_registry->delete_objects($curAgent);
				print "Deleted agent ".$curAgent->id.".($drop)\n" if $verbose > 2;
				next;
			}
			
		}
		for my $i (0..$#{$disinfo->{CLEAN}->{lines}}) {
			
			my ($keys, $fors) = MpmasQLFunctionLanguage->translate_and_interprete( $disinfo->{CLEAN}->{lines}->[$i],\%locvars1);
			for my $j (0..$#{$keys}) {		
			    $curAgent->drop_attribute($keys->[$j]);
			}
		}
	}
	print "Number of agents after cleaning in STEP $d:". $total_agent_registry->counter()."\n" if $verbose;
	print "\e[K \r";
	my $fh;
	for my $i (0..$#{$disinfo->{OUTPUT}->{lines}}) {
		my $x = $disinfo->{OUTPUT}->{lines}->[$i];
		$x =~ s/^\s*(\w+)//;
		my $what = $1;
		
		my $fn = "STDOUT";	
		if (not $x =~ /^\s*$/) {
			$fn = MpmasQLFunctionLanguage->convert($x,{});
			open ($fh, ">", $fn) or print STDERR "ERROR when trying to open $fn: $!\n" and exit(1);	
		}
		else {
			open ($fh, ">&STDOUT");	
		}
		if ($what eq "ASSIGNMENTS") {
			print "Writing ASSIGNMENTS to $fn\n";
			for my $ag ($total_agent_registry->object_list()) {
				 my %assets_consolidated;
				map { $assets_consolidated{$_->get_attribute("type") } += $_->get_attribute("size") } $ag->get_attribute("assigned")->object_list() ;
				map { print $fh join("\t", $ag->id, $_, $assets_consolidated{$_} )."\n"	} keys %assets_consolidated;
			}	
		}
		elsif ($what eq "MEMBERS") {
			print "Writing MEMBERS to $fn\n";
			for my $ag ($total_agent_registry->object_list()) {
				 my %members_consolidated;
				map { $members_consolidated{$_->get_attribute("type") } += $_->get_attribute("number") } $ag->get_attribute("memberlist")->object_list() ;
				map { print $fh join("\t", $ag->id, $_, $members_consolidated{$_} )."\n"	} keys %members_consolidated;
			}	
		}
		elsif ($what eq "NOTASSIGNED") {
			print "Writing NOTASSIGNED to $fn\n";
			for my $asset ($notassigned->object_list()) {	
				next unless(ref $asset);
				print $fh $asset->get_attribute("id");
				for my $att ( $asset->attribute_list) {
					next if ($att eq "id");
					print $fh "\t".$att."=".$asset->get_attribute($att);
					
				}	
				print $fh "\n";
			}	
		}
		elsif ($what eq "AGENTS") {
			print "Writing AGENTS to $fn\n";
			for my $agent ($total_agent_registry->object_list()) {
				next unless(ref $agent);
				for my $att ( sort{$a cmp $b} $agent->attribute_list) {
					next if (is_in_list($att,[("id", "assigned")]));
					print $fh join("\t", $agent->get_attribute("id"),$att,$agent->get_attribute($att))."\n";
				}	
				
			}	
		}
		elsif ($what eq "STORE_AGENTS") {
			print "Storing AGENTS to $fn\n";
			$total_agent_registry->save($fn);	
		}
		elsif ($what eq "STORE_NOTASSIGNED") {
			print "Storing NOTASSIGNED to $fn\n";
			$notassigned->save($fn);	
		}
		elsif ($what eq "STORE_SUBAREAS") {
			print "Storing SUBAREAS to $fn\n";
			$subarearegistry->save($fn);	
		}
		elsif ($what =~ /(FARMSTEAD|PROPERTY|SOIL|CLUSTER|POPULATION|NETWORK|SECTOR|CATCHMENT)MAP/) {
			my $which = $1;
			print "Writing ${which}MAP to $fn\n";
			print STDERR "Error: No ${which}MAP has been generated yet!\n" unless (defined $mapregistry->get_object_by_id("${which}MAP"));
			$mapregistry->get_object_by_id("${which}MAP")->save_as_ascii($fn);	
		}

	
		close ($fh);
	}
	($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
	$mon += 1;
	$year += 1900;
	print "\e[K \rSTEP $d DISTRIBUTION COMPLETE\tTIME: $year $mon $mday , $hour:$min:$sec\n";
}



