#!/usr/bin/perl
use strict;
use warnings;
use Test::More;
use JSON;
use Data::Dumper;
use File::Spec;
use File::Path;
use lib qw(/usr/local/lib/mpmasql/modules);
use MpmasInputfile;
use MpmasMILP;
use IndexedTable;
use MpmasQlEqnTranslatorMarpa;
use CtUtilities qw/get_unique/;


#(c) Christian Troost, Universität Hohenheim, 2023-24


##Version:

my $thisversion = "0.72";
my $lastchange = "14 Feb 2024";


my $starttime = time;

my $longwelcome = <<WC;

	Model testing  for MPMAS 
	Version: $thisversion, last change: $lastchange
	(c) Troost, C., Univ. Hohenheim, 2023-24		
	
	
WC

my $shortwelcome = <<WC2;
	test_mpmas v$thisversion		
WC2

#my $OStype = $^O ;

our $list_entries = {
	
	1 =>  { 
	 	mip => { required  => {
		 				scenario => 1,
						expected => 1,	 			
		 				title	=>  1,
		 				what	=> 1	 	
	 		 		},
	 		   relevant => {	paramvar_scenario => 1,
	 		   			constraint => 1,
	 		   			activity   => 1,
	 		   			precision => 1,
	 		   			comparison => 1
	 		   		},
		},
		result_wide => { required  => {
		 				scenario => 1,
						expected => 1,	 			
		 				title	=>  1,
	 		   			activity   => 1,
	 		 		},
	 		   relevant => {	paramvar_scenario => 1,	 		   			
	 		   			precision => 1,
	 		   			comparison => 1
	 		   		},
   		},
	 	result_long => { required  => {
		 				scenario => 1,
						expected => 1,	 			
		 				title	=>  1,
	 		   			activity   => 1,
	 		 		},
	 		   relevant => {	paramvar_scenario => 1,	 		   			

	 		   			precision => 1,
	 		   			comparison => 1
	 		   		},
   		},
		aggresult => { required  => {
		 				scenario => 1,
						expected => 1,	 			
		 				title	=>  1,
		 				output_variable => 1,
	 		 		},
	 		   relevant => {	paramvar_scenario => 1,	 		   			

	 		   			precision => 1,
	 		   			comparison => 1
	 		   		},
   		},
   		conversion_error => {  required  => {
		 				scenario => 1,		
		 				title	=>  1
	 		 		},
	 		   relevant => {	message => 1,
	 		   			exactly => 0	 		   			
	 		   		},
   		}

	},
	0 => { 
	 	mip => { required  => {
		 				scenario => 1,
						expected => 1,	 			
		 				title	=>  1,
		 				what	=> 1,
		 				agent => 1,
		 				period => 1	 	
	 		 		},
	 		   relevant => {	paramvar_scenario => 1,
	 		   			constraint => 1,
	 		   			activity   => 1,
	 		   			precision => 1,
	 		   			decisionStage => 1,
	 		   			comparison => 1
	 		   		},
		},
		result => { required  => {
		 				scenario => 1,
						expected => 1,	 			
		 				title	=>  1,
	 		   			activity   => 1,
		 				agent => 1,
		 				period => 1
	 		 		},
	 		   relevant => {	paramvar_scenario => 1,		   			
	 		   			precision => 1,
	 		   			decisionStage => 1,
	 		   			comparison => 1
	 		   		},
   		},

		aggresult => { required  => {
		 				scenario => 1,
						expected => 1,	 			
		 				title	=>  1,
		 				output_variable => 1,
		 				agent => 1,
		 				period => 1
	 		 		},
	 		   relevant => {			   			
						paramvar_scenario => 1,
	 		   			precision => 1,
	 		   			decisionStage => 1,
	 		   			comparison => 1
	 		   		},
   		},
	 	aggresult_by_year => { required  => {
		 				scenario => 1,
						expected => 1,	 			
		 				title	=>  1,
		 				output_variable => 1,
		 				agent => 1,
		 				period => 1
	 		 		},
	 		   relevant => {	paramvar_scenario => 1,		   			
	 		   			precision => 1,
	 		   			decisionStage => 1,
	 		   			comparison => 1
	 		   		},
 		 },
 		 ,
   		conversion_error => {  required  => {
		 				scenario => 1,		
		 				title	=>  1
	 		 		},
	 		   relevant => {	message => 1,
	 		   			exactly => 0	 		   			
	 		   		},
   		} ,
   		runtime_error => {  required  => {
		 				scenario => 1,		
		 				title	=>  1
	 		 		},
	 		   relevant => {	paramvar_scenario => 1,
	 		   			message => 1,
	 		   			exactly => 0	 		   			
	 		   		},
   		}
	},


};

my $input_fileset = "./";
my @testfiles;
my $skiprun = 0;


##Process command line arguments
while (@ARGV) {
	my $element = shift @ARGV;
	if  ($element =~ /^-v$/) {
		print $longwelcome;
		exit 0;
	}
	if  ($element =~ /^--skip-run$/) { 

		$skiprun = 1; 
	}
	elsif (not $element =~ /^-/) {
		# inifile name
		push @testfiles, $element;
	}
	elsif ($element =~ /^-h/ or $element =~ /^--help/ or $element =~ /^-\?/) {
		print <<HELP;
		
Usage:
	test_mpmas <filenames> [options]
	
	<filename>	list of .json files with tests

Options:
	--skip-run	don't run mpmasql/MPMAS but work with existing output files
		
	-v		print version information
	-h, --help, -?	print this help
	
HELP

		exit (0);
	}
	else {
		print STDERR "Error: Unknown commandline argument $element\nCall with flag --help for usage information\n" ;
		exit(1);
	}
}


##Loop over all test files
my $json_fh;
for my $testfile (@testfiles) {

	my $standalone = 0;
	# Read the JSON data from the file
	open ($json_fh, '<', $testfile) or (print STDERR "Unable to open file: $!" and exit(1));
	my $json_text = do { local $/; <$json_fh> };
	close $json_fh;

	# Parse the JSON data into a Perl data structure
	my $jsondata;
	
	eval {
		$jsondata = decode_json($json_text);
	};
	if ($@) {
		my $t = __FILE__;
		my $mes = $@;
		$mes =~ s/at $t line \d+.\s*$//;
		print STDERR "Error while reading JSON data in '$testfile':\n$mes\n";
		exit(1)
	}

	my $ini_file_name;	
	unless (exists $jsondata->{"ini"}) {
		print STDERR "Error: No .ini file specified in $testfile.";
		exit(1);
	}
	$ini_file_name = $jsondata->{"ini"};
	
	if (exists $jsondata->{"model_type"} ) {
		if( $jsondata->{"model_type"} eq "standalone" ) {
			$standalone = 1;
		}
		elsif ( $jsondata->{"model_type"} eq "full" ) {
			$standalone = 0;
		}
	}
	
	

	my $input_dat_dir = "input/dat/";
	my $mpmas_out_dir = "out/";

	$input_dat_dir = $jsondata->{"mpmas_input_dir"}  if (exists $jsondata->{"mpmas_input_dir"});
	$mpmas_out_dir = $jsondata->{"mpmas_out_dir"}  if (exists $jsondata->{"mpmas_out_dir"});
	

	
	
	my $checklist = $jsondata->{"tests"};

	#creating checklist hash index from checklist array to open files only once for more efficient processing.
	my $checkhash = {};

	for my $c (@$checklist) {

		check_consistency_of_test_entries($c, $standalone, $testfile);
		unless (exists $c->{comparison} && defined ($c->{comparison})) {
			if (defined $c->{what} and ($c->{what} eq "description" or $c->{what} eq "unit")) {
				$c->{comparison} = "eq"	;		
			}
			else {
				$c->{comparison} = "==";
			}
		}

		if ($c->{outfile_type} eq "conversion_error") {
			if  ( defined $c->{message} && not defined $c->{exactly}) {
				$c->{exactly} = 0;
			}		
			if (exists $checkhash->{$c->{outfile_type} }{$c->{scenario} } ) {
				print STDERR "Error in $testfile: More than one  conversion_Error entries for scenario ".$c->{scenario}. ".\nOnly one conversion error should be tested per scenario.\n"; exit(1);
			}
			$checkhash->{$c->{outfile_type} }{$c->{scenario} } = $c;
		
		}
		elsif ($c->{outfile_type} eq "runtime_error") {
			if  ( defined $c->{message} && not defined $c->{exactly}) {
				$c->{exactly} = 0;
			}
			my $scen =  $c->{scenario} ;
			if (exists $c->{paramvar_scenario}) {
			
				if (exists $checkhash->{$c->{outfile_type} }{$c->{scenario} }{$scen}  ) {
					print STDERR "Inconsistent runtime_error entries for ".{$c->{scenario} }."\nListed with and without paramvar_scenario.\n";
					
					exit(1);		
				}
			
			
				$scen .= "_".$c->{paramvar_scenario} 
			}
			elsif (exists $checkhash->{$c->{outfile_type} }{$c->{scenario} }  ) {
				print STDERR "Inconsistent runtime_error entries for ".{$c->{scenario} }."\nlisted multiple times and at least once without paramvar_scenario.\n";
				
				exit(1);		
			}
			
			if (exists $checkhash->{$c->{outfile_type} }{$c->{scenario} }{$scen}) {
				print STDERR "Error in $testfile: More than one  runtime_error entries for scenario ".$scen. ".\nOnly one runtime error should be tested per scenario unless multiple parameter variation scenarios are used.\nIn the latter case, only one runtime error should be tested per scenario and parameter variation scenario.\n"; 
				exit(1);
			}
			$checkhash->{$c->{outfile_type} }{$c->{scenario} }{$scen} = $c;
			
		}

			
			
		 
		
		 
		else {
			my $scen =  $c->{scenario} ;
			if (exists $c->{paramvar_scenario}) {
				$scen .= "_".$c->{paramvar_scenario} 
			}
			if (  $standalone ) {
				unless (exists $checkhash->{$c->{outfile_type} }{$c->{scenario}}{$scen }) {
					$checkhash->{$c->{outfile_type} }{$c->{scenario}}{$scen }  = []
				}
				push @{ $checkhash->{ $c->{outfile_type} }{$c->{scenario}} {$scen } }  ,  $c;
			}
			else {
				my $d = ( $c->{outfile_type} eq "mip") ? "StartOfYear": "periodSwitch";
				
				if (exists $c->{decisionStage}) {
					$d = $c->{decisionStage} 
				}

				
				
				unless (exists $checkhash->{$c->{outfile_type} }{$c->{scenario}}{$scen } {$c->{period}} {$d} {$c->{agent}}) {
					$checkhash->{$c->{outfile_type} }{$c->{scenario}}{$scen } {$c->{period}} {$d} {$c->{agent}} = []
				}
				push @{$checkhash->{$c->{outfile_type} }{$c->{scenario}}{$scen  } {$c->{period}} {$d} {$c->{agent}}} ,  $c;
			}
		}
	}

	#print Dumper($checkhash);
	chdir ($input_fileset) or die $!;
	my %successlines;
	my %errorlines;
	
	
	my $runtime = time();
	my ($testfilevol, $testfiledirs, $testfilefn) = File::Spec->splitpath($testfile);
	

	&mkpath("testlogs");
	if (exists $checkhash->{conversion_error} && scalar(keys %{$checkhash->{conversion_error}}) > 0 ) {
	
		
		print "Running conversion_error tests:\n";
		for my $scenCrash  ( keys %{$checkhash->{conversion_error}} ) {
		
			my $logfile = "testlogs/$testfilefn.$scenCrash.mpmasql_run.log";
			
			unlink ("$logfile") ;

			my $rtcod = system("mpmasql $ini_file_name --prefixed-scens $scenCrash 1> $logfile 2>&1");
;
			ok($rtcod > 0, $scenCrash.": to exit with return code > 0");

			
			my $openlogcheck = open (my $fh, "<", "$logfile" ) ; 


			ok($openlogcheck, "$logfile created.");
			exit(1) unless ($openlogcheck);

			my $lastline2;
			while (<$fh>) {
				my $temp =  $_;
				next if ($temp =~ /^\s*$/);
				$lastline2 = $temp;
				
			}
			#print $lastline2;
			if (defined $checkhash->{conversion_error}->{$scenCrash}->{message}) {
				my $ex = $checkhash->{conversion_error}->{$scenCrash}->{message};
				unless ($checkhash->{conversion_error}->{$scenCrash}->{exactly}) {

					like( $lastline2, qr/$ex/ , $scenCrash.": Error message.");
				}
				else {
					ok( $lastline2 eq  $ex , $scenCrash.": Exact error message.");
				}
				
			}
			close ($fh);
		}
		print "\nRunning expected results tests:\n";

		#then run all other scenarios
		my @otherScens;
		for my $ft ( keys %{$list_entries->{$standalone}} ) {
			next if ($ft eq "conversion_error");
			push @otherScens, keys %{$checkhash->{$ft}} if (exists $checkhash->{$ft});
		}
		my $logfile = "testlogs/$testfilefn.mpmas_run.log";
		my $scenstring = join (",", get_unique(@otherScens));
		unless ($skiprun) {
			unlink ("$logfile") ;
			system("mpmasql --test-format --prefixed-scens $scenstring --run-and-wait $ini_file_name  --set \"ONLY_DEFINED_SCENARIOS=1\" 1> $logfile 2>&1");

			my $openlogcheck = open (my $fh, "<", "$logfile" ) ; 


			ok($openlogcheck, "$logfile created.");
			exit(1) unless ($openlogcheck);

			my $curLogScen = "";
			while (<$fh>) {
				my $temp =  $_;
				next if ($temp =~ /^\s*$/);
				
				if ($temp =~ /^\s+Scenario name\s+:\s+(\w+)\s+/) {
					$curLogScen = $1;
					;
				
				}
				elsif ($temp =~ /\s*(Error|ERROR)/) {
					$errorlines{$curLogScen} .= $temp;
				}
				elsif ($temp =~ /^\s*Simulation experiment\[${curLogScen}(_)? .+\] completed/) {
					$successlines{$curLogScen} = "completed";
				
				}
				
				
			}
			#print $lastline;
			close ($fh);
		}		
	}
	else {
		my $logfile = "testlogs/$testfilefn.mpmasql_run.log";
		unless ($skiprun) {
			unlink ("$logfile") ;
			system("mpmasql --test-format --run-and-wait $ini_file_name  1> $logfile 2>&1");

			my $openlogcheck = open (my $fh, "<", "$logfile" ) ; 


			ok($openlogcheck, "$logfile created.");
			exit(1) unless ($openlogcheck);


			my $curLogScen = "";
			while (<$fh>) {
				my $temp =  $_;
				next if ($temp =~ /^\s*$/);
				
				if ($temp =~ /^\s+Scenario name\s+:\s+(\w+)\s+/) {
					$curLogScen = $1;
					;
				
				}
				elsif ($temp =~ /(Error|ERROR)/) {
					$errorlines{$curLogScen} .= $temp;
				}
				elsif ($temp =~ /^\s*Simulation experiment\[${curLogScen} .+\] completed/) {
					$successlines{$curLogScen} = "completed";
				
				}
				
				
			}
			#print $lastline;
			close ($fh);
				}
	}
	



	##FOR FULL DYNAMIC MODELS
	if (!$standalone) {
	

		for my $s (keys %{$checkhash->{runtime_error} }   ) {

  		     for my $s2 (keys %{$checkhash->{runtime_error}->{$s} }   ) {			

				if (ok (exists $errorlines{$s2."_"}, "$s2 to create runtime error" )) {
					
					if (defined $checkhash->{runtime_error}->{$s}{$s2}->{message}) {
						my $ex = $checkhash->{runtime_error}->{$s}{$s2}->{message};
						unless ($checkhash->{runtime_error}->{$s}{$s2}->{exactly}) {

							like( $errorlines{$s2."_"}, qr/$ex/ , $checkhash->{runtime_error}->{$s}{$s2}->{title}.": Error message.");
						}
						else {
							ok( $errorlines{$s2."_"} eq  $ex , $checkhash->{runtime_error}->{$s}{$s2}->{title}.": Exact error message.");
						}
				
					}
				}
		     }
		}
		my @otherScens;
		for my $ft ( keys %{$list_entries->{$standalone}} ) {
			next if ($ft eq "conversion_error" || $ft eq "runtime_error" );
			if (exists $checkhash->{$ft}) {
				for my $s (keys %{$checkhash->{$ft}} ) {
					push @otherScens, keys %{$checkhash->{$ft}{$s}} 
				}
			}
		}
		
		#check whether other simulations finished successfully
		for my $s (@otherScens) {
			ok( $successlines{$s."_"} , "Simulation $s finished successfully.");# unless ($skiprun);
		}



		#process all MIP content checks
		for my $s (keys %{$checkhash->{mip} }   ) {
			#print "$s\n";
  		     for my $s2 (keys %{$checkhash->{mip}->{$s} }   ) {			
			for my $p (keys %{$checkhash->{mip}->{$s}{$s2} }   ) {
			#print "$s $p\n";
			  for my $d (keys %{$checkhash->{mip}->{$s}{$s2} ->{$p} }   ) {
				#print "$s $p $d\n";
				for my $ag (keys %{$checkhash->{mip}->{$s}{$s2} ->{$p}->{$d} }   ) {
					#print "$s $p $d $ag\n";
					my $scen = $s2;
					#$scen =~ s/_$//;
					my $filenameGlob;

					$filenameGlob = $input_fileset . "/".$mpmas_out_dir."/test/" . $scen. "_". $p."_".$ag. $d."*MIP*[0-9].mpx";

					my @files = glob $filenameGlob;
					unless (scalar @files  == 1 ) {
						print STDERR "ERROR: Stopping prematurely due to missing or ambiguous file for $filenameGlob.\n";
						exit (1);
					}
					unless (-e $files[0] and ($skiprun == 1  || (stat($files[0]))[9]  >= $runtime)) {  
						print STDERR "File $files[0] does not exist or is older than start of this test\n"  ;
						exit (1);
					}
					my $c1 = $checkhash->{mip}->{$s}{$s2} ->{$p}->{$d}->{$ag}[0];
					my $actdesname = $input_fileset . "/".$input_dat_dir ."/" . $c1->{scenario} . "_"."ActivityInfo.txt"; 
					my $condesname = $input_fileset . "/".$input_dat_dir ."/" . $c1->{scenario} . "_"."ConstraintInfo.txt";
				
					my $matrix = MpmasMILP->recreate_from_file($files[0], "Failed", $actdesname, $condesname,1, undef, 0);
					for my $c (   @{$checkhash->{mip}->{$s}{$s2} ->{$p}->{$d}->{$ag} } ) {		
						my $description = "S: ". $s2. ", Ag: ".$c->{agent}. ", P: ".$c->{period}. ", DS: ".$d ;
						check_matrix_content($matrix, $c, $description);

					} ## end loop over entries
				} ## end loop over agents
			  } ## end loop over decision stages
			} ## end loop over periods
				
		  }## end loop over combined scenario/parameter_scenario

		} ##end loop over scenarios
		
		#process all checks on raw results
		for my $s (keys %{$checkhash->{result} }   ) {
		  for my $s2 (keys %{$checkhash->{result}->{$s}  }   ) {
			for my $p (keys %{$checkhash->{result}->{$s}{$s2}  }   ) {
				for my $d (keys %{$checkhash->{result}->{$s}{$s2}->{$p} }   ) {


					my $scen = $s2;
					#$scen =~ s/_$//;
					my $filenameGlob;
					$filenameGlob = $input_fileset . "/".$mpmas_out_dir."/" . $scen. "__AgentDecisionResultsVec_".$d."_MIP1000_".sprintf("%02d", $p).".out";
					
					my @files = glob $filenameGlob;
					unless (scalar @files  == 1 ) {
						print STDERR "ERROR: Stopping prematurely due to missing or ambiguous file for $filenameGlob.\n";
						exit (1);
					}
					unless (-e $files[0] and ($skiprun  or  (stat($files[0]))[9]  >= $runtime)) {  
						print STDERR "File $files[0] does not exist or is older than start of this test\n"  ;
						exit (1);
					}
					
					
					my $resultData = IndexedTable->create_from_ascii($files[0],  5);
					
					
					for my $ag (keys %{$checkhash->{result}->{$s}{$s2}->{$p}->{$d} }   ) {
					   for my $c (   @{$checkhash->{result}->{$s}{$s2}->{$p}->{$d}->{$ag} } ) {		
						$c->{expected} = sprintf("%.".$c->{precision}."f",$c->{expected}) if (exists $c->{precision});	
						my $x  = $resultData->query_by_columnname([$s2."_", $p, $d, $ag, $c->{activity}], "Solution");
						$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});
						cmp_ok( $x, $c->{comparison}, $c->{expected}, "S: ". $s2.", Ag: ".$c->{agent}. ", P: ".$c->{period}. ", DS: ".$d. ", Solution for " .$c->{activity}   ) ;
					   }					
					}
				}

			}
		  }
		}
		
		#process all checks on aggregated results
		for my $ot ("aggresult", "aggresult_by_year") {
			for my $s (   (keys %{$checkhash->{$ot}})   ) {#scenarios
  			  for my $s2 (keys %{$checkhash->{$ot }->{$s} }   ) {#periods
				for my $p (keys %{$checkhash->{$ot }->{$s} {$s2}}   ) {#periods
					for my $d (keys %{$checkhash->{$ot }->{$s}{$s2}->{$p} }   ) {#decisionStages


						my $scen = $s2;
						
						#$scen =~ s/_$//;
						my $filenameGlob;
							
						if ($ot eq "aggresult_by_year") {
						   $filenameGlob = $input_fileset . "/".$mpmas_out_dir."/" . $scen. "__AgentDecisionResultsAgg_".$d."_MIP1000_".sprintf("%02d", $p).".out";
						}
						if ($ot eq "aggresult") {
						   $filenameGlob = $input_fileset . "/".$mpmas_out_dir."/" . $scen. "__AgentDecisionResultsAgg_".$d."_MIP1000.out";
						}
						
						my @files = glob $filenameGlob;
						unless (scalar @files  == 1 ) {
							print STDERR "ERROR: Stopping prematurely due to missing or ambiguous file for $filenameGlob.\n";
							exit (1);
						}
						unless (-e $files[0] and ($skiprun  or (stat($files[0]))[9]  >= $runtime)) {  
							print STDERR "File $files[0] does not exist or is older than start of this test\n"  ;
							exit (1);
						}
							
						
						my $resultData = IndexedTable->create_from_ascii($files[0],  4);
						
						
						for my $ag (keys %{$checkhash->{$ot }->{$s}{$s2}->{$p}->{$d} }   ) {
						   for my $c (   @{$checkhash->{$ot }->{$s}{$s2}->{$p}->{$d}->{$ag} } ) {		
							$c->{expected} = sprintf("%.".$c->{precision}."f",$c->{expected}) if (exists $c->{precision});	
							
							unless (ok($resultData->exists_columnname($c->{output_variable}), "Column '".$c->{output_variable}."' must exist.")) {
								print STDERR "ERROR: Stopping prematurely due to missing outputfile column.\n";
								exit (1);
							}
							
							my $x  = $resultData->query_by_columnname([$scen."_", $p, $d, $ag], $c->{output_variable});
							$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});
							cmp_ok( $x, $c->{comparison}, $c->{expected}, "S: ". $scen.", Ag: ".$c->{agent}. ", P: ".$c->{period}. ", DS: ".$d. ", Agg. result for " .$c->{output_variable}   ) ;
						   }					
						}
					}

				}
			}
		 }	
		}

		
		
		
	}
	##FOR STATIC MODELS
	elsif ($standalone) {
		#process all MIP content checks
		for my $s (keys %{$checkhash->{mip} }   ) {
			#print "$s\n";
		   for my $s2 (  keys %{$checkhash->{mip}->{$s}} ) {		
			for my $c (   @{$checkhash->{mip}->{$s}{$s2} } ) {		
				my $scen = $s2;
				#$scen =~ s/_$//;
				my $filenameGlob ;
				my $description; 
				my $mxmode;
				if (exists $c->{paramvar_scenario}) {
					$filenameGlob = $input_fileset . "/".$input_dat_dir ."/". $c->{scenario}. "_MILP_".$c->{paramvar_scenario}.".tst";
					$description =  "S: ". $c->{scenario}. ", PVS: ". $c->{paramvar_scenario}  ;
					$mxmode = "Failed";
				}
				else {
					$filenameGlob = $input_fileset . "/".$input_dat_dir . "/".$c->{scenario}. "_MILP.tst";
					$description =  "S: ". $c->{scenario};
					$mxmode = "Failed";
				}
				my @files = glob $filenameGlob;
				unless (scalar @files  == 1 ) {
					print STDERR "ERROR: Stopping prematurely due to missing or ambiguous file for $filenameGlob.\n";
					exit (1);
				}
				unless (-e $files[0] and ($skiprun  or stat($files[0]))[9]  >= $runtime) {  
					print STDERR "File $files[0] does not exist or is older than start of this test\n"  ;
					exit (1);
				}
					


				my $actdesname = $input_fileset . "/".$input_dat_dir ."/" . $c->{scenario} . "_"."ActivityInfo.txt"; 
				my $condesname = $input_fileset . "/".$input_dat_dir ."/" . $c->{scenario} . "_"."ConstraintInfo.txt";

				my $matrix = MpmasMILP->recreate_from_file($files[0], $mxmode, $actdesname, $condesname,1, undef, 0);


				check_matrix_content($matrix, $c, $description);
			

			} ##end loop over scenarios
		    }
		} ##end loop over mip
		
		#process all checks on raw results if in wide format
		for my $s (keys %{$checkhash->{result_wide} }   ) {
			for my $s2 (keys %{$checkhash->{result_wide}{$s} }   ) {

					my $scen = $s2;
					#$scen =~ s/_$//;
					
					my $c1 = $checkhash->{result_wide}->{$s}{$s2}[0];
					my $filenameGlob = $input_fileset . "/".$input_dat_dir."/" . $c1->{scenario}. "_MILP_Results.txt";
					my @files = glob $filenameGlob;
					unless (scalar @files  == 1 ) {
						print STDERR "ERROR: Stopping prematurely due to missing or ambiguous file for $filenameGlob.\n";
						exit (1);
					}
					unless (-e $files[0] and ($skiprun  or stat($files[0]))[9]  >= $runtime) {  
						print STDERR "File $files[0] does not exist or is older than start of this test\n"  ;
						exit (1);
					}
							
					
					
					my $resultData = IndexedTable->create_from_ascii($files[0],  2);
					
					

					   for my $c (   @{$checkhash->{result_wide}->{$s}{$s2} } ) {		
						$c->{expected} = sprintf("%.".$c->{precision}."f",$c->{expected}) if (exists $c->{precision});	

						unless (ok($resultData->exists_columnname($c->{activity}), "Column '".$c->{activity}."' must exist.")) {
							print STDERR "ERROR: Stopping prematurely due to missing outputfile column.\n";
							exit (1);
						}
						my $pv = "default";
						my $filestub = ($resultData->query([],0))[0];
						$pv = $c->{paramvar_scenario} if (exists ($c->{paramvar_scenario}));	
						my $x  = $resultData->query_by_columnname([$filestub, $pv], $c->{activity});

						$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});
						cmp_ok( $x, $c->{comparison}, $c->{expected}, "S: ". $s.", PVS: ".$pv. ", Solution for " .$c->{activity}   ) ;
					   }					
					
			}
		}
		
		#process all checks on aggregated results if in wide format
		for my $s (keys %{$checkhash->{aggresult} }   ) {
			for my $s2 (keys %{$checkhash->{aggresult}{$s} }   ) {

					my $scen = $s2;
					#$scen =~ s/_$//;
					my $c1 = $checkhash->{aggresult}->{$s}{$s2}[0];
					my $filenameGlob = $input_fileset . "/".$input_dat_dir."/" . $c1->{scenario}. "_MILP_AggResults.txt";
					my @files = glob $filenameGlob;
					unless (scalar @files  == 1 ) {
						print STDERR "ERROR: Stopping prematurely due to missing or ambiguous file for $filenameGlob.\n";
						exit (1);
					}
					unless (-e $files[0] and ($skiprun  or stat($files[0]))[9]  >= $runtime) {  
						print STDERR "File $files[0] does not exist or is older than start of this test\n"  ;
						exit (1);
					}
					
					
					my $resultData = IndexedTable->create_from_ascii($files[0],  2);
					
					

					   for my $c (   @{$checkhash->{aggresult}->{$s}{$s2} } ) {		
						$c->{expected} = sprintf("%.".$c->{precision}."f",$c->{expected}) if (exists $c->{precision});	

						unless (ok($resultData->exists_columnname($c->{output_variable}), "Column '".$c->{output_variable}."' must exist.")) {
							print STDERR "ERROR: Stopping prematurely due to missing outputfile column.\n";
							exit (1);
						}
						my $pv = "default";
						my $filestub = ($resultData->query([],0))[0];
						$pv = $c->{paramvar_scenario} if (exists ($c->{paramvar_scenario}));	
						my $x  = $resultData->query_by_columnname([$filestub, $pv], $c->{output_variable});

						$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});
						cmp_ok( $x, $c->{comparison}, $c->{expected}, "S: ". $c->{scenario}.", PVS: ".$pv. ", Solution for " .$c->{output_variable}   ) ;
					   }					
			}		

		}

		#process all checks on raw results if in long format
		for my $s (keys %{$checkhash->{result_long} }   ) {
			for my $s2 (keys %{$checkhash->{result_long}{$s} }   ) {

					my $scen = $s2;
					#$scen =~ s/_$//;
					my $c1 = $checkhash->{result_long}->{$s}{$s2}[0];
					my $filenameGlob = $input_fileset . "/".$input_dat_dir."/" . $c1->{scenario}. "_MILP_Results.txt";
					my @files = glob $filenameGlob;
					unless (scalar @files  == 1 ) {
						print STDERR "ERROR: Stopping prematurely due to missing or ambiguous file for $filenameGlob.\n";
						exit (1);
					}
					unless (-e $files[0] and ($skiprun  or stat($files[0]))[9]  >= $runtime) {  
						print STDERR "File $files[0] does not exist or is older than start of this test\n"  ;
						exit (1);
					}
					
					
					my $resultData = IndexedTable->create_from_ascii($files[0],  3);
					
					

					   for my $c (   @{$checkhash->{result_long}->{$s}{$s2} } ) {		
						$c->{expected} = sprintf("%.".$c->{precision}."f",$c->{expected}) if (exists $c->{precision});	

						#unless (ok($resultData->exists_columnname($c->{activity}), "Column '".$c->{activity}."' must exist.")) {
						#	print STDERR "ERROR: Stopping prematurely due to missing outputfile column.\n";
						#	exit (1);
						#}
						my $pv = "default";

						my $filestub = ($resultData->query([],0))[0];
						$pv = $c->{paramvar_scenario} if (exists ($c->{paramvar_scenario}));	
						my $x  = $resultData->query_by_columnname([$filestub, $pv, $c->{activity}], "Solution" );
						$x  = 0 unless (defined $x);
						$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});
						
						cmp_ok( $x, $c->{comparison}, $c->{expected}, "S: ". $c->{scenario}.", PVS: ".$pv. ", Solution for " .$c->{activity}   ) ;
					   }					
					
			}
		}
		



	}#end if standalone	
	
	
	

} ##loop over files

done_testing();




sub check_matrix_content {
		my ($matrix, $c,  $description) = @_;
		
		$c->{expected} = sprintf("%.".$c->{precision}."f",$c->{expected}) if (exists $c->{precision});				
		
		if ( $c->{what} eq "coefficient"   ) {
			print STDERR "Error in test specification: activity missing for test ".$c->{title}."\n" and exit(1) unless (exists $c->{activity});
			print STDERR "Error in test specification: constraint missing for test ".$c->{title}."\n" and exit(1) unless (exists $c->{constraint});
		
			my $k = $matrix->activity($c->{activity});
			print STDERR "Error: Activity ".$c->{"activity"}. " does not exist\n" and exit(1) unless (defined $k);
			
			my $test = $matrix->constraint($c->{constraint});
			print STDERR "Error: Activity ".$c->{"constraint"}. " does not exist\n" and exit(1) unless (defined $test);

			
			my $x = $k->coeff($c->{constraint});
			$x = 0 unless (defined $x);					
			$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});

			cmp_ok( $x, $c->{comparison}, $c->{expected}, $description. ", MIP Coefficient ".$c->{activity}." : ".$c->{constraint}   ) ;
		}
		elsif ( $c->{what} eq "lbound" || $c->{what} eq "ubound"   ) {
			print STDERR "Error in test specification: activity missing for test ".$c->{title}."\n" and exit(1) unless (exists $c->{activity});
			my $k = $matrix->activity($c->{activity});
			print STDERR "Error: Activity ".$c->{"activity"}. " does not exist\n" and exit(1) unless (defined $k);
			my $x = $k->attr($c->{what});
			$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});
			cmp_ok( $x, $c->{comparison}, $c->{expected}, $description. ", MIP ".$c->{what}."  ".$c->{activity}   ) ;
		}
		elsif ( $c->{what} eq "integ" ||  $c->{what} eq "integer"  ) {
			print STDERR "Error in test specification: activity missing for test ".$c->{title}."\n" and exit(1) unless (exists $c->{activity});
			my $k = $matrix->activity($c->{activity});
			print STDERR "Error: Activity ".$c->{"activity"}. " does not exist\n" and exit(1) unless (defined $k);
			my $x = $k->attr("integ");
			$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});
			cmp_ok( $x, $c->{comparison}, $c->{expected}, $description. ", MIP ".$c->{what}."  ".$c->{activity}   ) ;
		}	
		elsif ( $c->{what} eq "objective"  ) {
			print "Error in test specification: activity missing for test ".$c->{title}."\n" and exit(1) unless (exists $c->{activity});
			my $k = $matrix->activity($c->{activity});
			print STDERR "Error: Activity ".$c->{"activity"}. " does not exist\n" and exit(1) unless (defined $k);
			my $x = $k->orow();
			$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});
			cmp_ok( $x, $c->{comparison}, $c->{expected}, $description. " MIP OFcoeff. ".$c->{activity}   ) ;
		}	
		elsif ( $c->{what} eq "rhs"  ) {

			my $k = $matrix->constraint($c->{constraint});
			print STDERR "Error: Constraint ".$c->{"constraint"}. " does not exist\n" and exit(1) unless (defined $k);
			my $x = $k->attr("value");
			$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});
			cmp_ok( $x, $c->{comparison}, $c->{expected}, $description. ", MIP RHS value: ".$c->{constraint}   ) ;
		}
		elsif ( $c->{what} eq "description" || $c->{what} eq "unit") {

			print STDERR "Error in test specification: both activity and constraint given  for test ".$c->{title}."\nUnclear which ".$c->{what} . "to test\n" and exit(1) if (exists $c->{constraint} and exists $c->{activity});

			print STDERR "Error in test specification: Missing activity or constraint  for test ".$c->{title}."\n" and exit(1) if (not exists $c->{constraint} and not exists $c->{activity});

			my $k;
			if (exists $c->{constraint}) {
				$k = $matrix->constraint($c->{constraint});
				print STDERR "Error: Constraint ".$c->{"constraint"}. " does not exist\n" and exit(1) unless (defined $k);
			}
			elsif(exists $c->{activity}) {
				$k = $matrix->activity($c->{activity});
				print STDERR "Error: Activity ".$c->{"activity"}. " does not exist\n" and exit(1) unless (defined $k);
			}
			
			my $x; 
			if ($c->{what} eq "description") {
				$x = $k->name();			
			}
			elsif ($c->{what} eq "unit") {
				$x = $k->attr("unit" );
			}
			$x = sprintf("%.".$c->{precision}."f",$x) if (exists $c->{precision});

			cmp_ok( $x, $c->{comparison}, $c->{expected}, $description. ", MIP ".$c->{what}.": ". (exists ($c->{constraint}) ? $c->{constraint} : $c->{activity})   ) ;
		}
		else {
			print STDERR "Error in test specification: unknown 'what' = ".$c->{what}." for test ".$c->{title}."\n" ;
			exit(1);
		}

}

BLOCK_FOR_CHECKING: {

  sub check_consistency_of_test_entries {
	
	my ($c, $standalone, $testfile) = @_;
	
		unless (exists $c->{outfile_type}) {
			print Dumper($c);
			print STDERR "Error: outfile_type missing in $testfile\n";
			(exists $c->{title})  ? (print STDERR $c->{title}) : (print STDERR "untitled test");
			print "\n";
			
			exit(1);     
		}

		unless (exists $list_entries->{$standalone}{$c->{outfile_type}}   ){
			print Dumper($c);
			print STDERR "Error: outfile_type ".$c->{outfile_type}. " unknown for ".  ($standalone ? " standalone " : "full dynamic") . " models in $testfile\n";
			print "\n";
			exit(1);       
		}

		for my $i (keys %{ $list_entries->{$standalone}{$c->{outfile_type}}{required}}) {

			unless (exists ($c->{$i})  ) {
				print Dumper($c);
				print STDERR "Error: Incomplete test information in $testfile: '$i' is required for outfile_type '".$c->{outfile_type}."'".
					($standalone ? " in standalone mode" : " in dynamic models")."\n";		
				
				(exists $c->{title})  ? (print STDERR $c->{title}) : (print STDERR "untitled test");
				print "\n";

				exit(1); 
			}
		}
		
		
		for my $i ( keys %$c) {
			next if ($i eq "outfile_type");
			unless ( exists  $list_entries->{$standalone}{$c->{outfile_type}}{required}{$i} or exists $list_entries->{$standalone}{$c->{outfile_type}}{relevant}{$i}   )  {
				print Dumper($c);
				print STDERR "Error: Inconsistent test information in $testfile: '$i' is not relevant for outfile_type '".$c->{outfile_type}."'".
					($standalone ? " in standalone mode" : " in dynamic models")."\n";
				(exists $c->{title})  ? (print STDERR $c->{title} ) : (print STDERR "untitled test");
				print "\n";


				exit(1); 
			
			}
		}
	}
}




##OLD TEST DATA
#my $checklist =[
#	{id=> "implicit addition check", outfile_type => "mip", scenario => "basic20_multi_baseline_", agent=> 1, period => 1, activity => "xSellHarvestYear0_maize_1", constraint => "cashBalanceStartOfYear_1" , what => "coefficient",
#	expected =>  - 0.123 + (-1.5 * 262)   },
#
#	{id=> "inserter agent.cash_reserves", outfile_type => "mip", scenario => "basic20_multi_baseline_", agent=> 2, period => 0, what => "rhs",  
#	constraint => "cashBalanceStartOfYear_1" , 	expected =>  850   },
#	{id=> "plain value", outfile_type => "mip", scenario => "basic20_multi_baseline_", agent=> 1, period => 1, activity => "xSellHarvestYear0_maize_1", what => "lbound" , 	expected =>  0   },
#	{id=> "plain value", outfile_type => "mip", scenario => "basic20_multi_baseline_", agent=> 5, period => 3, activity => "xInvestInIrrigationPump_3", what => "integer" , 	expected =>  1   },
#		{id=> "plain value", outfile_type => "mip", scenario => "basic20_multi_baseline_", agent=> 1, period => 1, activity => "xInvestInIrrigationPump_1", what => "ubound" , 	expected =>  100   },
#	{id=> "plain value", outfile_type => "mip", scenario => "basic20_multi_baseline_", agent=> 4, period => 9, activity => "xValueTerminalValues", what => "objective" , 	expected =>  1 / (1+0.06)**15  , precision => 5 },
#		
#];


1;

