hierlearning/0000775000175000017500000000000012304226737013413 5ustar nevillenevillehierlearning/launch.pl0000664000175000017500000001704312303746004015220 0ustar nevilleneville#!/usr/bin/perl ################################################################################ # Runs a script to scan for an open port, runs Stratagus and tells it to open # # the socket interface on the open port, then runs your client and tells it to # # connect to this port. # ################################################################################ ########################## BEGIN CONFIGURABLE OPTIONS ######################### # Change to the directory containing the Stratagus executable my $STRATAGUS_DIR = '../other/stratagus'; # Extra arguments to pass to Stratagus (the data directory, the port, and # the map arguments are already passed so there's no need to include them here) my $STRATAGUS_OPTS = '-l'; # Modify this if the data directory for Stratagus is in a non-standard location my $STRATAGUS_DATA = "${STRATAGUS_DIR}/data"; # The location of the map relative to $STRATAGUS_DATA my $STRATAGUS_MAP = ''; # Modify this if you want to change the default port where scanning starts my $START_PORT = 4870; # Maximum numbers of ports to try before giving up my $MAX_PORTS = 100; # Xvfb executable chop(my $XVFB = `which Xvfb 2> /dev/null`); # xauth executable chop(my $XAUTH = `which xauth 2> /dev/null`); # Client command-line where any instance of the string "_PORT_" will be # replaced by the actual port before the command is run (providing this option # on the command line will override whatever you have here) my $CLIENT_CMD = ''; ########################## END CONFIGURABLE OPTIONS ########################### use Fcntl; use Fcntl ':flock'; use File::Temp ':mktemp'; use Getopt::Std; use IO::Socket; use POSIX; $0 =~ m|.*/(.*)|; my $progname = $1; sub usage { print STDERR << "EOF"; Usage: $progname [-g] [-m map-file] [client-command-line] -g By default, we run a virtual X server (requires Xvfb to be installed) and connect Stratagus to it so no graphics are displayed. Use this option if you actually want to display the graphics. -m map-file The path of the map relative to the data directory. client-command-line Command line used to run your client. Any instance of the string "_PORT_" will be replaced by the actual port that Stratagus used to open the socket interface. This option is required unless you specify a default value at the top of the script. Example: If the command to run your client is "./client -p port" and you want to run Stratagus with the map map.pud.gz loaded, use the following command: $progname -m maps/map.pud.gz ./client -p _PORT_ EOF exit; } my $locked = 0; my $xvfb_pid; my $stratagus_pid; my $show_graphics = 0; my $xserver_num; my $xauthority; my $xauth_added = 0; my $lock_file_handle; my $lock_filename = "/tmp/transfer-lock"; sub get_lock { sysopen($lock_file_handle, "$lock_filename", O_RDONLY | O_CREAT) or die "Failed to open $lock_filename: $!"; chmod(0666, $lock_filename); my $retries = 60; while (!flock($lock_file_handle, LOCK_EX | LOCK_NB)) { if ($retries-- == 0) { die "Failed to obtain file lock on $lock_filename: $!"; } sleep(1); } $locked = 1; } sub release_lock { close($lock_file_handle) or die "Failed to close $lock_filename: $!"; $locked = 0; } # Clean up END { system("kill -9 $stratagus_pid 2> /dev/null") if $stratagus_pid > 0; system("kill $xvfb_pid 2> /dev/null") if $xvfb_pid > 0; system("$XAUTH -f $xauthority remove :$xserver_num") if $xauth_added; unlink($xauthority) if $xauth_added; release_lock() if $locked; if ($xvfb_pid || $stratagus_pid) { print "\nClean-up successful.\n"; } } # Make sure our cleanup function is called when we exit use sigtrap qw(handler END normal-signals error-signals); # Parse command-line options my %options = (); getopts("hgm:", \%options); # Print usage if the -h option is given or we don't have the info we need usage() if defined $options{h}; usage() if !$ARGV[0] and !$CLIENT_CMD; $show_graphics = 1 if defined $options{g}; if (!$show_graphics) { if (not -e "$XAUTH") { die "Can't find xauth executable; try running with -g option"; } elsif (not -e "$XVFB") { die "Can't find Xvfb executable; try running with -g option"; } } # Use Stratagus map from command line if specified $STRATAGUS_MAP = $options{m} if defined $options{m}; if (not -e "${STRATAGUS_DATA}/${STRATAGUS_MAP}") { die "Can't find Stratagus map: ${STRATAGUS_DATA}/${STRATAGUS_MAP}"; } # Check for the Stratagus executable if (not -e "${STRATAGUS_DIR}/stratagus") { die "Can't find Stratagus executable: ${STRATAGUS_DIR}/stratagus"; } # Check for the client executable if (not -e "$ARGV[0]") { die "Can't find client executable: $ARGV[0]"; } # Checking client command for "_PORT_" string if (!$CLIENT_CMD) { $CLIENT_CMD = join(" ", @ARGV); } die "Client command must contain _PORT_" unless $CLIENT_CMD =~ /_PORT_/; # Clean up dead children without wait()ing $SIG{'CHLD'} = 'IGNORE'; # Only allow one instance of the script to scan for ports at a time get_lock(); # Scan ports until we find an open one my $socket; $MAX_PORTS = $START_PORT + $MAX_PORTS; for ( ; $START_PORT < $MAX_PORTS; $START_PORT++) { $socket = new IO::Socket::INET(LocalHost => 'localhost', LocalPort => $START_PORT, Proto => 'tcp', ReuseAddr => 1); # If socket connection succeeded (port is free), break out of the loop last if $socket; } die "Failed to find an open port" if $START_PORT == $MAX_PORTS; close($socket); # Block SIGINT while forking $sigset = POSIX::SigSet->new(SIGINT); sigprocmask(SIG_BLOCK, $sigset) or die "Can't block SIGINT for fork: $!"; # Run Xvfb if needed if (!$show_graphics) { # Find the first free X server number $xserver_num = 100; while (-f "/tmp/.X${xserver_num}-lock" || -e "/tmp/.X11-unix/X$xserver_num") { $xserver_num++; } $xauthority = mktemp("/tmp/transfer.XXXXXX"); $ENV{'XAUTHORITY'} = $xauthority; $xvfb_pid = fork(); if ($xvfb_pid == 0) { # This is the child -- run Xvfb open(STDOUT, '>/dev/null'); open(STDERR, '>/dev/null'); system("$XAUTH -f $xauthority add :$xserver_num . `mcookie`"); exec "${XVFB} :$xserver_num"; } else { print "Running Xvfb (with Xauthority)\n"; } die "Fork failed: $!" unless $xvfb_pid; $xauth_added = 1; } # Run Stratagus $stratagus_pid = fork(); if ($stratagus_pid == 0) { # This is the child -- run Stratagus open(STDOUT, '>/dev/null'); open(STDERR, '>/dev/null'); $ENV{'DISPLAY'} = "localhost:${xserver_num}.0" if !$show_graphics; exec "${STRATAGUS_DIR}/stratagus -d $STRATAGUS_DATA -p $START_PORT $STRATAGUS_OPTS $STRATAGUS_MAP"; } else { print "Running Stratagus: ${STRATAGUS_DIR}/stratagus -d $STRATAGUS_DATA -p $START_PORT $STRATAGUS_OPTS $STRATAGUS_MAP\n"; } die "Fork failed: $!" unless $stratagus_pid; # Unblock SIGINT sigprocmask(SIG_UNBLOCK, $sigset) or die "Can't unblock SIGINT for fork: $!"; # Wait until Stratagus has opened the port and is accepting connections print "Waiting for the Stratagus server...\n"; $socket = 0; $retries = 60; while (!$socket && $retries-- > 0) { $socket = new IO::Socket::INET(PeerAddr => "localhost:$START_PORT", Proto => 'tcp', Timeout => '1'); sleep(1); } die "Tired of waiting for Stratagus" unless $socket; close($socket); print "Stratagus server ready.\n"; release_lock(); # Replacing _PORT_ with the value of $START_PORT $CLIENT_CMD =~ s/_PORT_/$START_PORT/; # Run the client print "Running client: $CLIENT_CMD\n"; system($CLIENT_CMD); exit; hierlearning/hierlearning.vcxproj.filters0000664000175000017500000002526012303746004021144 0ustar nevilleneville {4FC737F1-C7A5-4376-A066-2A32D752A2FF} cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx {9e046b7b-1b71-46b9-b9e5-eb33cd196766} {99ff5de3-c83a-4fd4-aeb6-7a9c9eeb6c84} {3a9ca35c-afd8-4ecc-91d7-2b1d64d75cd7} {62e8871c-f165-474b-9d75-335a47b90bdb} {f2c893a6-c9d1-4962-8f1c-96627cfe192f} {f115d054-edca-4b77-baed-df8af59cb5ed} {edb39f90-2ffd-4c72-831c-5b4e3556471b} {58f17055-337b-4426-b9fe-924e580ebdbc} {2e5e0e2d-65c2-423f-b871-28ce8aa3b128} {ba79d84a-b176-4e9b-be85-65b2cd01562c} {c9a9fd53-4626-4871-b812-0e03ddb14bf3} {83388563-d05f-467b-a3ea-a598b589e94d} {acd97881-820d-449d-ab59-48b789d2df13} {405e45c7-e59d-451f-a4fe-cec80f81c85d} {53cce233-40ff-49eb-8b8c-69d43fc6a797} {78913302-fdc3-401c-8fd2-ad0ddc593a6d} src src lib domain\taxi domain\taxi domain\taxi domain\taxi domain\bitflip domain\wargus domain\wargus domain\wargus domain\wargus\client hiergen hiergen hiergen hiergen hiergen learner\hier\maxq learner\hier\maxq learner\hier\maxq learner\hier\hh learner\hier\hh learner\hier\hh learner\hier\hh learner\hier\hh learner\hier\generic learner\hier\generic learner\hier\generic learner\hier\generic learner\flat\q learner\flat\generic learner\flat\generic learner\flat\h src lib lib lib domain domain domain\taxi domain\taxi domain\taxi domain\taxi domain\bitflip domain\wargus domain\wargus domain\wargus domain\wargus\client learner learner hiergen hiergen hiergen hiergen hiergen learner\flat\q learner\flat learner\hier\maxq learner\hier\maxq learner\hier\maxq learner\hier\maxq learner\hier\hh learner\hier\hh learner\hier\hh learner\hier\hh learner\hier\hh learner\hier\generic learner\hier\generic learner\hier\generic learner\hier\generic learner\hier\generic learner\flat\q learner\flat\generic learner\flat\generic learner\flat\h learner\flat\h learner\hier hiergen learner\hier\generic hierlearning/hiergen/0000775000175000017500000000000012304226346015030 5ustar nevillenevillehierlearning/hiergen/expression.cc0000664000175000017500000010006412205065452017536 0ustar nevilleneville /************************************************* EXPRESSION Neville Mehta **************************************************/ #include #include #include "../domain/mdp.h" #include "expression.h" Expression::Expression (const string& expression, const State& state, const set& relevant_variables) { string filtered_expression = replace(expression, " ", ""); if (filtered_expression.empty()) throw HierException(__FILE__, __LINE__, "Expression string is empty."); expression_parse(filtered_expression, state, relevant_variables); } void Expression::expression_parse (const string& expression, const State& state, const set& relevant_variables) { if (expression.find(",") != string::npos) { exop = And; value = 0; vector expression_tokens = tokenize(expression, ","); for (const auto& expression_token : expression_tokens) { Expression temp_expression(expression_token, state, relevant_variables); if (relevant_variables.empty() || relevant_variables.find(*temp_expression.lhs_variables().begin()) != relevant_variables.end()) children.push_back(temp_expression); } if (children.size() == 1) { Expression temp_expression(children.front()); *this = temp_expression; // Cannot directly assign the children due to memory issues } } else if (expression.find_first_of("<>!=") != string::npos) { string::size_type start = expression.find_first_not_of("<>!="); string::size_type end = expression.find_first_of("<>!=", start); if (expression.substr(end, 1) == "=") exop = Equal; else if (expression.substr(end, 2) == "<=") exop = Less_Equal; else if (expression.substr(end, 2) == ">=") exop = Greater_Equal; else if (expression.substr(end, 1) == "<") exop = Less; else if (expression.substr(end, 1) == ">") exop = Greater; else if (expression.substr(end, 2) == "!=") exop = Not_Equal; else throw HierException(__FILE__, __LINE__, "Invalid expression operator."); value = 0; children.emplace_back(expression.substr(start, end - start), state, relevant_variables); start = expression.find_first_not_of("<>=", end); end = expression.find_first_of("<>=", start); children.emplace_back(expression.substr(start, end - start), state, relevant_variables); if (end != string::npos) throw HierException(__FILE__, __LINE__, "Bad expression."); } else { pair parse_value = state.parse(expression); if (parse_value.first) { exop = Value; value = parse_value.second; } else { exop = Variable; value = state.variable_index(expression); } } } void Expression::generalized_condition (const vector>& states, const set& variables) { // Find the conditions for variables that are true for the entire set of states // The family of conditions a variable is: (1) equal to constant, (2) equal to another variable set all_variables = extract_map_keys(states.front()); set condition_vars = variables.empty() ? all_variables : variables; // Constant on r.h.s. for (auto var = condition_vars.cbegin(); var != condition_vars.cend(); ) { int value = states.front().find(*var)->second; bool equal_to_value = true; for (unsigned s = 1; s < states.size(); ++s) if (states[s].find(*var)->second != value) { equal_to_value = false; break; } if (equal_to_value) // var = value { conjunct(Expression(*var, false, value)); condition_vars.erase(var++); // No need to find any more conditions for this variable } else ++var; } // Variable on r.h.s. set> equal_variables; // Equivalence relation for (const auto& var1 : condition_vars) for (const auto& var2 : all_variables) if (var2 != var1) { set var_set = make_set(2, var1, var2); bool equal_to_variable = false; for (const auto& ev : equal_variables) if (includes(ev.begin(), ev.end(), var_set.begin(), var_set.end())) { equal_to_variable = true; break; } if (equal_to_variable) continue; equal_to_variable = true; for (const auto& state : states) if (state.find(var1)->second != state.find(var2)->second) { equal_to_variable = false; break; } if (equal_to_variable) // var1 = var2 { conjunct(Expression(var1, true, var2)); bool matching_block_found = false; for (auto ev_itr = equal_variables.cbegin(); ev_itr != equal_variables.cend(); ++ev_itr) if (ev_itr->find(var1) != ev_itr->end() || ev_itr->find(var2) != ev_itr->end()) { set block = *ev_itr; block.insert(var_set.begin(), var_set.end()); equal_variables.erase(ev_itr); equal_variables.insert(block); matching_block_found = true; break; } if (!matching_block_found) equal_variables.insert(var_set); } } } Expression::Expression (const int& lhs_variable_index, const bool& rhs_variable, const int& index_or_value) : exop(Equal), value(0) { children.emplace_back(Variable, lhs_variable_index); if (rhs_variable) children.emplace_back(Variable, index_or_value); else children.emplace_back(Value, index_or_value); } Expression::Expression (const vector& conditions) : exop(NoOp), value(0) { // Constructs a conjunction of factors that are present in all the conjunctive conditions if (conditions.empty()) return; vector factors = conditions.front().factorize(); for (const auto& factor : factors) { bool global_factor = true; for (const auto& condition : conditions) { if (condition.exop == Or) throw HierException(__FILE__, __LINE__, "Expression is a disjunction."); if (!condition.subsumes(factor)) { global_factor = false; break; } } if (global_factor) conjunct(factor); } } int Expression::evaluate (const State& state, const vector& parameters, const unsigned& agent) const { switch (exop) { case NoOp: throw HierException(__FILE__, __LINE__, "Nothing to evaluate."); case Value: return value; case Variable: return state.variable(value); case Agent_Variable: return state.variable(agent * state.num_agent_variables() + value); case Parameter: if (value >= (int)parameters.size()) throw HierException(__FILE__, __LINE__, "Expression parameter not found."); return parameters[value]; case Equal: return children.front().evaluate(state, parameters, agent) == children.back().evaluate(state, parameters, agent); case Less_Equal: return children.front().evaluate(state, parameters, agent) <= children.back().evaluate(state, parameters, agent); case Greater_Equal: return children.front().evaluate(state, parameters, agent) >= children.back().evaluate(state, parameters, agent); case Less: return children.front().evaluate(state, parameters, agent) < children.back().evaluate(state, parameters, agent); case Greater: return children.front().evaluate(state, parameters, agent) > children.back().evaluate(state, parameters, agent); case Not_Equal: return children.front().evaluate(state, parameters, agent) != children.back().evaluate(state, parameters, agent); case And: for (const auto& child : children) if (!child.evaluate(state, parameters, agent)) return 0; return 1; case Or: for (const auto& child : children) if (child.evaluate(state, parameters, agent)) return 1; return 0; default: throw HierException(__FILE__, __LINE__, "Unknown expression operator"); } } int Expression::evaluate (const map& variables_map) const { switch (exop) { case NoOp: throw HierException(__FILE__, __LINE__, "Nothing to evaluate."); case Value: return value; case Variable: { auto v_itr = variables_map.find(value); if (v_itr == variables_map.end()) throw HierException(__FILE__, __LINE__, "Expression variable not found."); return v_itr->second; } case Equal: return children.front().evaluate(variables_map) == children.back().evaluate(variables_map); case Less_Equal: return children.front().evaluate(variables_map) <= children.back().evaluate(variables_map); case Greater_Equal: return children.front().evaluate(variables_map) >= children.back().evaluate(variables_map); case Less: return children.front().evaluate(variables_map) < children.back().evaluate(variables_map); case Greater: return children.front().evaluate(variables_map) > children.back().evaluate(variables_map); case Not_Equal: return children.front().evaluate(variables_map) != children.back().evaluate(variables_map); case And: for (const auto& child : children) if (!child.evaluate(variables_map)) return 0; return 1; case Or: for (const auto& child : children) if (child.evaluate(variables_map)) return 1; return 0; default: throw HierException(__FILE__, __LINE__, "Unknown expression operator"); } } bool Expression::operator == (const Expression& expression) const { if (children.size() != expression.children.size()) return false; switch (exop) { case NoOp: return exop == expression.exop; case Value: case Variable: case Agent_Variable: case Parameter: return exop == expression.exop && value == expression.value; case Equal: case Not_Equal: return exop == expression.exop && ((children.front() == expression.children.front() && children.back() == expression.children.back()) || (children.front() == expression.children.back() && children.back() == expression.children.front())); case Less_Equal: return (exop == expression.exop && children.front() == expression.children.front() && children.back() == expression.children.back()) || (expression.exop == Greater_Equal && children.front() == expression.children.back() && children.back() == expression.children.front()); case Greater_Equal: return (exop == expression.exop && children.front() == expression.children.front() && children.back() == expression.children.back()) || (expression.exop == Less_Equal && children.front() == expression.children.back() && children.back() == expression.children.front()); case Less: return (exop == expression.exop && children.front() == expression.children.front() && children.back() == expression.children.back()) || (expression.exop == Greater && children.front() == expression.children.back() && children.back() == expression.children.front()); case Greater: return (exop == expression.exop && children.front() == expression.children.front() && children.back() == expression.children.back()) || (expression.exop == Less && children.front() == expression.children.back() && children.back() == expression.children.front()); case And: case Or: { if (exop != expression.exop) return false; // Accounting for the commutative property vector subexp_index; // Indexes the unmatched children of the rhs expression for (unsigned i = 0; i < expression.children.size(); ++i) subexp_index.push_back(i); bool matched = false; for (const auto& child : children) { matched = false; for (unsigned i = 0; i < subexp_index.size(); ++i) if (child == expression.children[subexp_index[i]]) { subexp_index.erase(subexp_index.begin() + i); // Removing matched child matched = true; break; } if (!matched) break; } if (matched || equivalence_equality(expression)) return true; return false; } default: throw HierException(__FILE__, __LINE__, "Unknown expression operator."); } } bool Expression::parameter_binding_equality (const Expression& expression, map& parameter_map) const { // Returns true if there exists an equality-making mapping from variables in this to the variables in expression if (children.size() != expression.children.size()) return false; switch (exop) { case NoOp: return exop == expression.exop; case Value: case Variable: case Agent_Variable: return exop == expression.exop && value == expression.value; case Parameter: if (expression.exop == Parameter) { map::const_iterator p_itr = parameter_map.find(value); if (p_itr != parameter_map.end()) return p_itr->second == value; else { parameter_map[value] = expression.value; return true; } } return false; case Equal: case Not_Equal: return exop == expression.exop && ((children.front().parameter_binding_equality(expression.children.front(), parameter_map) && children.back().parameter_binding_equality(expression.children.back(), parameter_map)) || (children.front().parameter_binding_equality(expression.children.back(), parameter_map) && children.back().parameter_binding_equality(expression.children.front(), parameter_map))); case Less_Equal: return (exop == expression.exop && children.front().parameter_binding_equality(expression.children.front(), parameter_map) && children.back().parameter_binding_equality(expression.children.back(), parameter_map)) || (expression.exop == Greater_Equal && children.front().parameter_binding_equality(expression.children.back(), parameter_map) && children.back().parameter_binding_equality(expression.children.front(), parameter_map)); case Greater_Equal: return (exop == expression.exop && children.front().parameter_binding_equality(expression.children.front(), parameter_map) && children.back().parameter_binding_equality(expression.children.back(), parameter_map)) || (expression.exop == Less_Equal && children.front().parameter_binding_equality(expression.children.back(), parameter_map) && children.back().parameter_binding_equality(expression.children.front(), parameter_map)); case Less: return (exop == expression.exop && children.front().parameter_binding_equality(expression.children.front(), parameter_map) && children.back().parameter_binding_equality(expression.children.back(), parameter_map)) || (expression.exop == Greater && children.front().parameter_binding_equality(expression.children.back(), parameter_map) && children.back().parameter_binding_equality(expression.children.front(), parameter_map)); case Greater: return (exop == expression.exop && children.front().parameter_binding_equality(expression.children.front(), parameter_map) && children.back().parameter_binding_equality(expression.children.back(), parameter_map)) || (expression.exop == Less && children.front().parameter_binding_equality(expression.children.back(), parameter_map) && children.back().parameter_binding_equality(expression.children.front(), parameter_map)); case And: case Or: { if (exop != expression.exop) return false; // Accounting for the commutative property vector subexp_index; // Indexes the unmatched children of the rhs expression for (unsigned i = 0; i < expression.children.size(); ++i) subexp_index.push_back(i); for (const auto& child : children) { bool matched = false; for (unsigned i = 0; i < subexp_index.size(); ++i) if (child.parameter_binding_equality(expression.children[subexp_index[i]], parameter_map)) { subexp_index.erase(subexp_index.begin() + i); // Removing matched child matched = true; break; } if (!matched) return false; } return true; } default: throw HierException(__FILE__, __LINE__, "Unknown expression operator."); } } bool Expression::equivalence_equality (const Expression& expression) const { vector lhs_subexp_index; // Indexes the unmatched children of the lhs expression for (unsigned i = 0; i < children.size(); ++i) lhs_subexp_index.push_back(i); vector rhs_subexp_index; // Indexes the unmatched children of the rhs expression for (unsigned i = 0; i < expression.children.size(); ++i) rhs_subexp_index.push_back(i); for (unsigned l = 0; l < lhs_subexp_index.size(); ) { if (children[lhs_subexp_index[l]].exop == Equal && children[lhs_subexp_index[l]].children.front().exop == Variable && children[lhs_subexp_index[l]].children.back().exop == Variable) { ++l; continue; } bool matched = false; for (unsigned r = 0; r < rhs_subexp_index.size(); ++r) if (children[lhs_subexp_index[l]] == expression.children[rhs_subexp_index[r]]) { lhs_subexp_index.erase(lhs_subexp_index.begin() + l); // Removing matched child in lhs rhs_subexp_index.erase(rhs_subexp_index.begin() + r); // Removing matched child in rhs matched = true; break; } if (!matched) ++l; } if (lhs_subexp_index.empty() && rhs_subexp_index.empty()) return true; Expression lhs(exop, value); for (const auto& index : lhs_subexp_index) lhs.children.push_back(children[index]); set> lhs_equivalence = lhs.equivalence(); if (lhs_equivalence.empty()) return false; Expression rhs(expression.exop, expression.value); for (const auto& index : rhs_subexp_index) rhs.children.push_back(expression.children[index]); set> rhs_equivalence = rhs.equivalence(); if (rhs_equivalence.empty()) return false; if (lhs_equivalence == rhs_equivalence) return true; return false; } set> Expression::equivalence () const { // Returns the equivalence relation set> equal_variables; for (const auto& child : children) if (child.exop == Equal && child.children.front().exop == Variable && child.children.back().exop == Variable) { bool matching_block_found = false; for (auto ev_itr = equal_variables.cbegin(); ev_itr != equal_variables.cend(); ++ev_itr) if (ev_itr->find(child.children.front().value) != ev_itr->end() || ev_itr->find(child.children.back().value) != ev_itr->end()) { set block = *ev_itr; block.insert(child.children.front().value); block.insert(child.children.back().value); equal_variables.erase(ev_itr); equal_variables.insert(block); matching_block_found = true; break; } if (!matching_block_found) equal_variables.insert(make_set(2, child.children.front().value, child.children.back().value)); } return equal_variables; } bool Expression::subsumes (const Expression& expression) const { // Returns true if the argument is part of this expression if (exop == And || exop == Or) { if (expression.exop == exop) { vector subexp_index; // Indexes the unmatched children of the rhs expression for (unsigned i = 0; i < expression.children.size(); ++i) subexp_index.push_back(i); for (unsigned i = 0; i < subexp_index.size(); ++i) { bool matched = false; for (const auto& child : children) if (child == expression.children[subexp_index[i]]) { subexp_index.erase(subexp_index.begin() + i); // Removing matched child matched = true; break; } if (!matched) return false; } return true; } else { for (const auto& child : children) if (child == expression) return true; } } else if (*this == expression) return true; return false; } set Expression::lhs_variables () const { if (children.empty()) return set(); set variables; if (exop == And || exop == Or) for (const auto& child : children) { set child_abstraction = child.lhs_variables(); variables.insert(child_abstraction.begin(), child_abstraction.end()); } else if (children.front().exop == Variable) variables.insert(children.front().value); return variables; } Expression Expression::lhs_expression () const { if (exop != Equal && exop != Less_Equal && exop != Greater_Equal && exop != Less && exop != Greater && exop != Not_Equal) throw HierException(__FILE__, __LINE__, "No lhs expression."); return children.front(); } set Expression::rhs_variables () const { if (children.empty()) return set(); set variables; if (exop == And || exop == Or) for (const auto& child : children) { set child_abstraction = child.rhs_variables(); variables.insert(child_abstraction.begin(), child_abstraction.end()); } else if (children.back().exop == Variable) variables.insert(children.back().value); return variables; } Expression Expression::rhs_expression () const { if (exop != Equal && exop != Less_Equal && exop != Greater_Equal && exop != Less && exop != Greater && exop != Not_Equal) throw HierException(__FILE__, __LINE__, "No rhs expression."); return children.back(); } set Expression::variables () const { set abstraction; if (children.empty()) { if (exop == Variable) abstraction.insert(value); } else for (const auto& child : children) { set child_abstraction = child.variables(); abstraction.insert(child_abstraction.begin(), child_abstraction.end()); } return abstraction; } int Expression::rhs_value () const { if (children.size() != 2 || children.back().exop != Value) throw HierException(__FILE__, __LINE__, "No rhs value."); return children.back().value; } bool Expression::variable_compared_to_constant (const int& variable) const { if (exop == Equal || exop == Less_Equal || exop == Greater_Equal || exop == Less || exop == Greater || exop == Not_Equal) if ((children.front().exop == Variable && children.front().value == variable && children.back().exop == Value) || (children.back().exop == Variable && children.back().value == variable && children.front().exop == Value)) return true; for (const auto& child : children) if (child.variable_compared_to_constant(variable)) return true; return false; } void Expression::parameterize (const vector& par_vars) { if (exop == Variable) { // Search for variable auto v_itr = find(par_vars.begin(), par_vars.end(), value); if (v_itr != par_vars.end()) { exop = Parameter; value = v_itr - par_vars.begin(); return; } } for (auto& child : children) child.parameterize(par_vars); } void Expression::set_agent_variables (const vector& variables) { if (variables.empty()) return; if (exop == Variable) { // Search for variable for (const auto& var : variables) if (var == value) { exop = Agent_Variable; return; } } for (auto& child : children) child.set_agent_variables(variables); } vector Expression::factorize () const { if (exop == And) { vector expression_factors; for (const auto& child : children) expression_factors.push_back(child); return expression_factors; } else return vector(); } void Expression::conjunct (const Expression& expression) { if (expression.exop == NoOp || *this == expression) return; switch (exop) { case NoOp: exop = expression.exop; value = expression.value; children = expression.children; return; case Value: case Variable: case Agent_Variable: case Parameter: case Equal: case Less_Equal: case Greater_Equal: case Less: case Greater: case Not_Equal: if (expression.exop == Or) // X & (X | Y) = X for (const auto& child : expression.children) if (*this == child) return; case Or: { // (X | Y) & X = X for (const auto& child : children) if (child == expression) { exop = expression.exop; value = expression.value; children = expression.children; return; } Expression temp_expression = *this; children.clear(); children.push_back(temp_expression); exop = And; value = 0; break; } case And: // (X & Y) & X = X & Y for (const auto& child : children) if (child == expression) return; break; } switch (expression.exop) { case NoOp: // Already checked, but here to placate gcc case Value: case Variable: case Agent_Variable: case Parameter: case Equal: case Less_Equal: case Greater_Equal: case Less: case Greater: case Not_Equal: case Or: children.push_back(expression); break; case And: for (const auto& child : expression.children) if (find(children.begin(), children.end(), child) == children.end()) children.push_back(child); break; } if (children.size() == 1) { Expression temp_expression = children.front(); *this = temp_expression; } } void Expression::disjunct (const Expression& expression) { if (expression.exop == NoOp || *this == expression) return; switch (exop) { case NoOp: exop = expression.exop; value = expression.value; children = expression.children; return; case Value: case Variable: case Agent_Variable: case Parameter: case Equal: case Less_Equal: case Greater_Equal: case Less: case Greater: case Not_Equal: if (expression.exop == And) // X | (X & Y) = X for (const auto& child : expression.children) if (*this == child) return; case And: { // (X & Y) | X = X for (const auto& child : children) if (child == expression) { exop = expression.exop; value = expression.value; children = expression.children; return; } Expression temp_expression = *this; children.clear(); children.push_back(temp_expression); exop = Or; value = 0; break; } case Or: // (X | Y) | Y = X | Y for (const auto& child : children) if (child == expression) return; break; } switch (expression.exop) { case NoOp: // Already checked, but here to placate gcc case Value: case Variable: case Agent_Variable: case Parameter: case Equal: case Less_Equal: case Greater_Equal: case Less: case Greater: case Not_Equal: case And: children.push_back(expression); break; case Or: for (const auto& child : expression.children) if (find(children.begin(), children.end(), child) == children.end()) children.push_back(child); break; } } void Expression::remove_factors (const Expression& expression) { if (exop != And || expression.empty()) return; if (expression.exop == And) { for (const auto& exp_child : expression.children) for (auto child_itr = children.begin(); child_itr != children.end(); ++child_itr) if (*child_itr == exp_child) { children.erase(child_itr); break; } } else { for (auto child_itr = children.begin(); child_itr != children.end(); ++child_itr) if (*child_itr == expression) { children.erase(child_itr); break; } } if (children.empty()) { exop = NoOp; return; } if (children.size() == 1) { Expression temp_expression(children.front()); *this = temp_expression; } } void Expression::ground_rhs_variables (const map& state) { switch (exop) { case NoOp: case Value: case Parameter: return; case Variable: exop = Value; value = state.find(value)->second; break; case Equal: case Less_Equal: case Greater_Equal: case Less: case Greater: case Not_Equal: children.back().ground_rhs_variables(state); break; case And: case Or: for (auto& child : children) child.children.back().ground_rhs_variables(state); break; default: throw HierException(__FILE__, __LINE__, "Illegal grounding."); } } vector Expression::exhaustive_grounding (const State& state) const { // Find all possible bindings of the variables on the rhs of the sub-expressions set variables = rhs_variables(); if (variables.empty()) return vector(1, *this); vector> variable_size; for (const auto& var : variables) variable_size.emplace_back(var, state.variable_size(var)); vector> possibilities; vector var(variable_size.size(), 0); do { possibilities.push_back(map()); for (unsigned i = 0 ; i < variable_size.size(); ++i) possibilities.back()[variable_size[i].first] = var[i]; var[0]++; for (unsigned i = 0; i < var.size() - 1 && var[i] == variable_size[i].second; ++i) { var[i] = 0; var[i+1]++; } } while (var.back() < (int)variable_size.back().second); if (possibilities.empty()) throw HierException(__FILE__, __LINE__, "No ground expressions found."); // Create the grounded expressions vector expression; for (const auto& possible : possibilities) { expression.push_back(*this); expression.back().ground_rhs_variables(possible); } return expression; } void Expression::negate () { switch (exop) { case NoOp: case Value: case Variable: case Agent_Variable: case Parameter: return; case Equal: exop = Not_Equal; break; case Less_Equal: exop = Greater_Equal; break; case Greater_Equal: exop = Less_Equal; break; case Less: exop = Greater; break; case Greater: exop = Less; break; case Not_Equal: exop = Equal; break; case And: exop = Or; break; case Or: exop = And; break; } for (auto& child : children) child.negate(); } void Expression::simplify () { for (auto& child : children) { child.simplify(); // Checking single operands for And/Or if ((child.exop == And || child.exop == Or) && child.children.size() == 1) child = child.children.front(); } switch (exop) { case NoOp: case Value: case Variable: case Agent_Variable: case Parameter: break; case Equal: case Less_Equal: case Greater_Equal: if (children.front() == children.back()) { exop = Value; value = 1; children.clear(); } break; case Less: case Greater: case Not_Equal: if (children.front() == children.back()) { exop = Value; value = 0; children.clear(); } break; case And: case Or: // True/false operands (0 And X = 0, 1 And X = X; 0 Or X = X, 1 Or X = 1) for (auto child_itr = children.begin(); child_itr != children.end(); ) { if (child_itr->exop == Value) { if (child_itr->value == (exop == And ? 0 : 1)) { exop = Value; value = exop == And ? 0 : 1; children.clear(); break; } else if (child_itr->value == (exop == And ? 1 : 0)) child_itr = children.erase(child_itr); else throw HierException(__FILE__, __LINE__, "Illegal value in expression."); } else ++child_itr; } // Replicated operands for (auto child_itr1 = children.begin(); child_itr1 != children.end(); ++child_itr1) { auto child_itr2 = child_itr1 + 1; while (child_itr2 != children.end()) { if (*child_itr1 == *child_itr2) child_itr2 = children.erase(child_itr2); else ++child_itr2; } } break; } } string Expression::print () const { switch (exop) { case NoOp: return ""; case Value: return to_string(value); case Variable: return "v" + to_string(value); case Agent_Variable: return "av" + to_string(value); case Parameter: return "x" + to_string(value); case Equal: return children.front().print() + " = " + children.back().print(); case Less_Equal: return children.front().print() + " <= " + children.back().print(); case Greater_Equal: return children.front().print() + " >= " + children.back().print(); case Less: return children.front().print() + " < " + children.back().print(); case Greater: return children.front().print() + " > " + children.back().print(); case Not_Equal: return children.front().print() + " != " + children.back().print(); case And: case Or: { auto child_itr = children.cbegin(); string expression_str = "(" + child_itr->print(); ++child_itr; for ( ; child_itr != children.cend(); ++child_itr) expression_str += (exop == And ? " & " : " | ") + child_itr->print(); expression_str += ")"; return expression_str; } default: throw HierException(__FILE__, __LINE__, "Unknown expression operator."); } } string Expression::print_html () const { switch (exop) { case NoOp: return ""; case Value: return to_string(value); case Variable: return "v" + to_string(value) + ""; case Agent_Variable: return "av" + to_string(value) + ""; case Parameter: return "x" + to_string(value) + ""; case Equal: return children.front().print_html() + " = " + children.back().print_html(); case Less_Equal: return children.front().print_html() + " <= " + children.back().print_html(); case Greater_Equal: return children.front().print_html() + " >= " + children.back().print_html(); case Less: return children.front().print_html() + " < " + children.back().print_html(); case Greater: return children.front().print_html() + " > " + children.back().print_html(); case Not_Equal: return children.front().print_html() + " != " + children.back().print_html(); case And: case Or: { auto child_itr = children.cbegin(); string expression_str = "(" + child_itr->print_html(); ++child_itr; for ( ; child_itr != children.cend(); ++child_itr) expression_str += (exop == And ? " & " : " | ") + child_itr->print_html(); expression_str += ")"; return expression_str; } default: throw HierException(__FILE__, __LINE__, "Unknown expression operator."); } } hierlearning/hiergen/model.h0000664000175000017500000000502412205065452016301 0ustar nevilleneville /************************************************* MODEL INTERFACE Neville Mehta **************************************************/ #pragma once #include #include #include #include "../domain/mdp.h" #include "expression.h" class ModelTree { vector> choices; int leaf; ModelTree(list& model_file_lines, const State& state, const int& level = 0); void read_file(list& model_file_lines, const State& state, const int& level); // Weka v3.6.5 J48 tree format bool is_leaf () const { return choices.empty(); } inline int true_choice(const State& state) const; inline int true_choice(const map& variable_value) const; string print_gv_cluster(const string& node_prefix, int& id) const; public: ModelTree(const int& leaf = 0) : leaf(leaf) {} ModelTree(const string& filename, const State& state); int get_value(const State& state) const; int get_value(const map& variable_value) const; set leaf_values() const; bool is_persistent(const int& variable) const; set checked_variables() const; set context_checked_variables(const map& variable_value) const; string print_gv_cluster (const string& node_prefix) const { int id = 0; return print_gv_cluster(node_prefix, id); } ~ModelTree () {} }; class ActionModel { struct SingleActionModel { map transition; set transition_checked_variables; ModelTree reward; set reward_checked_variables; set reward_checked_variables_depth_2; }; map _model; void read(const string& directory, const MDP& mdp); void compute_checked_variables(); set closure_next_stage(const pair& am, const set& closure_variables) const; void print_context_precondition(const MDP& mdp) const; public: ActionModel(const string& trajectory_filename, const string& model_directory, const MDP& mdp); ActionModel(const vector>>& input_trajectories, const string& model_directory, const unsigned& run_index, const MDP& mdp); set checked_variables(const int& action) const; set reward_variables(const int& action) const; set context_checked_variables(const int& action, const map& variable_value) const; set context_changed_variables(const int& action, const map& variable_value) const; set closure(const set& actions, const set& variables) const; void print_gv(const string& directory, const MDP& mdp) const; }; hierlearning/hiergen/expression.h0000664000175000017500000000625612205065452017410 0ustar nevilleneville /************************************************* EXPRESSION Neville Mehta **************************************************/ #pragma once #include #include #include #include "../lib/common.h" struct State; class Expression { enum Exop {NoOp = -1, Value, Variable, Agent_Variable, Parameter, Equal, Less_Equal, Greater_Equal, Less, Greater, Not_Equal, And, Or}; Exop exop; // Expression operator int value; // Leaf vector children; void expression_parse(const string& expression, const State& state, const set& relevant_variables); void generalized_condition(const vector>& states, const set& variables); void ground_rhs_variables(const map& state); bool variable_compared_to_constant(const int& variable) const; public: Expression(const Exop& exop = NoOp, const int& value = 0, const vector& children = vector()) : exop(exop), value(value), children(children) {} Expression(const string& expression, const State& state, const set& relevant_variables = set()); Expression(const vector>& states, const int& variable) : exop(NoOp), value(0) { generalized_condition(states, make_set(1, variable)); } Expression(const vector>& states, const set& variables = set()) : exop(NoOp), value(0) { generalized_condition(states, variables); } Expression(const int& lhs_variable_index, const bool& rhs_variable, const int& index_or_value); // Quick equation: "variable = variable" or "variable = value" Expression(const vector& conditions); int evaluate(const State& state, const vector& parameters = vector(), const unsigned& agent = 0) const; int evaluate(const map& variables_map) const; bool empty () const { return exop == NoOp; } void clear () { exop = NoOp; value = 0; children.clear(); } bool operator == (const Expression& expression) const; bool operator != (const Expression& expression) const { return !(*this == expression); } bool parameter_binding_equality(const Expression& expression, map& parameter_map) const; bool equivalence_equality(const Expression& expression) const; set> equivalence() const; bool subsumes(const Expression& expression) const; set lhs_variables() const; Expression lhs_expression() const; set rhs_variables() const; Expression rhs_expression() const; int rhs_value () const; bool equality () const { return exop == Equal; } set variables() const; // Get all variables void parameterize(const vector& variables); void set_agent_variables(const vector& variables); vector factorize() const; void conjunct(const Expression& expression); // Conjunction of two expressions void disjunct(const Expression& expression); // Disjunction of two expressions void remove_factors(const Expression& expression); // Remove the factors in expression from this conjunction vector exhaustive_grounding(const State& state) const; // Ground the values of the variables (all possibilities) void negate(); void simplify(); string print() const; string print_html() const; ~Expression () {} }; hierlearning/hiergen/trajectory.h0000664000175000017500000000660712205065452017377 0ustar nevilleneville /************************************************* CAUSALLY ANNOTATED TRAJECTORY Neville Mehta **************************************************/ #pragma once #include #include #include #include #include #include "../domain/mdp.h" #include "model.h" struct Subtrajectory { set action_indices; // Action indices of the actions within a trajectory map preconditions; // variable -> action index set outgoing_variables; Subtrajectory (const set& outgoing_variables = set()) : outgoing_variables(outgoing_variables) {} bool empty () const { return action_indices.empty(); } void clear () { action_indices.clear(); } }; class Trajectory { enum Special_Actions {Start_Action = -2, End_Action}; struct Trj_Node { map state; // State in the trajectory before the action is executed int action; double reward; struct Trj_Arc { map incoming; // Variable id, temporal index from trajectory node map> outgoing; // Variable id, temporal indices of trajectory nodes Trj_Arc() {} }; Trj_Arc causal_arcs; // Multiple arcs; each arcs has multiple labels Trj_Node (const map& state, const int& action, const double& reward = 0.0) : state(state), action(action), reward(reward) {} }; vector _trajectory; set incoming_variables(const set& action_indices) const; set outgoing_variables(const set& action_indices) const; public: Trajectory(istream& in, const MDP& mdp, const ActionModel& action_model); Trajectory(const vector>& sample_trajectory, const ActionModel& action_model); Trajectory(const Trajectory& trajectory, const set& action_indices); unsigned size () const { return _trajectory.size(); } void causal_annotation(const ActionModel& action_model, const set& goal_variables, const set& transfer_variables); void relevance_annotation(const ActionModel& action_model); void cleanup(); map extract_goal_state () const { return _trajectory.back().state; } map extract_state (const int& action_index) const { return _trajectory[action_index].state; } set extract_actions() const; int extract_action (const int& action_index) const { return _trajectory[action_index].action; } set extract_actions (const set& action_indices) const; set checked_variables() const; set changed_variables() const; set incoming_variables (const int& action_index) const { return incoming_variables(make_set(1, action_index)); } set outgoing_variables (const int& action_index) const { return outgoing_variables(make_set(1, action_index)); } set goal_variables () const { return incoming_variables(_trajectory.size() - 1); } int causal_action_index(const int& action_index, const int& variable) const; unique_ptr goal() const; unique_ptr extract_subtrajectory(const set& action_indices) const; unique_ptr extract_subtrajectory(const int& action_index) const { return extract_subtrajectory(make_set(1, action_index)); } unique_ptr splitter() const; bool always_true(const Expression& exp) const; map,double> cumulative_reward(const set& action_indices) const; void print_gv(const string& filename, const MDP& mdp) const; ~Trajectory () {} }; hierlearning/hiergen/catscan.cc0000664000175000017500000005065412205065452016764 0ustar nevilleneville /************************************************* CAUSALLY ANNOTATED TRAJECTORY SCANNER Neville Mehta **************************************************/ #include #include #include #include "../lib/common.h" #include "catscan.h" CatScan::CatScan (const vector& trajectories, const Expression& termination) : trajectories(trajectories) { // Add the preconditions of End action subtrj_matrix.emplace_back(trajectories.size(), termination); for (unsigned t = 0; t < trajectories.size(); ++t) subtrj_matrix.back().sub_trj[t] = trajectories[t].goal(); process(); cleanup(); // Trajectory split if (!decomposed()) // Only Root or with single child { if (subtrj_matrix.size() == 2) subtrj_matrix.pop_back(); // Remove single child subtrj_matrix.emplace_back(trajectories.size(), Expression(), 0); bool successful_split = true; for (unsigned t = 0; t < trajectories.size(); ++t) { subtrj_matrix.back().sub_trj[t] = trajectories[t].splitter(); if (!subtrj_matrix.back().sub_trj[t]) { successful_split = false; break; } } if (successful_split) { process(); cleanup(); } else subtrj_matrix.pop_back(); } if (decomposed()) precedence_graph_builder(); } bool CatScan::process () { map var_precondition; // variable -> precondition if (preconditions(var_precondition)) return true; if (var_precondition.empty()) return false; // Admissibility restricted to the abstraction variables process_admissibility(extract_map_keys(var_precondition)); // Process preconditions vector new_subtrj_matrix; vector> merger; for (const auto& var_prec : var_precondition) { new_subtrj_matrix.emplace_back(trajectories.size(), var_prec.second, subtrj_matrix.size() - 1); merger.emplace_back(trajectories.size()); for (unsigned t = 0; t < trajectories.size(); ++t) if (subtrj_matrix.back().sub_trj[t]) { map::const_iterator p_itr = subtrj_matrix.back().sub_trj[t]->preconditions.find(var_prec.first); if (p_itr != subtrj_matrix.back().sub_trj[t]->preconditions.end()) { int causal_action_index = trajectories[t].causal_action_index(p_itr->second, var_prec.first); if (causal_action_index > 0) { int subtrj_index; if ((subtrj_index = check_subtrajectories(causal_action_index, t))) // Subtrajectory match merger.back()[t].set(subtrj_index, false); else if ((subtrj_index = check_subtrajectories(new_subtrj_matrix, causal_action_index, t)) != -1) { new_subtrj_matrix.back().sub_trj[t] = move(new_subtrj_matrix[subtrj_index].sub_trj[t]); merger.back()[t].set(subtrj_index, true); } else { new_subtrj_matrix.back().sub_trj[t] = trajectories[t].extract_subtrajectory(causal_action_index); merger.back()[t].set_current(); } } } } } process_termination(new_subtrj_matrix, merger); const vector original_subtrj_matrix(subtrj_matrix); set bad_subtrj_columns; if (unify(new_subtrj_matrix, extract_map_keys(var_precondition), merger)) { for (unsigned k = 0; k < new_subtrj_matrix.size(); ++k) if (add_column(new_subtrj_matrix[k], merger[k]) && !process()) bad_subtrj_columns.insert(subtrj_matrix.size() - 1); } else { for (const auto& nsm : new_subtrj_matrix) { subtrj_matrix.push_back(nsm); bad_subtrj_columns.insert(subtrj_matrix.size() - 1); } } if (bad_subtrj_columns.empty() || (merge(bad_subtrj_columns) && process())) return true; // Reset the matrix subtrj_matrix = original_subtrj_matrix; return false; } bool CatScan::preconditions (map& var_precondition) const { bool empty_states = true; set pre_vars = precondition_variables(subtrj_matrix.size() - 1); for (const auto& var : pre_vars) { vector> states; for (unsigned t = 0; t < trajectories.size(); ++t) if (subtrj_matrix.back().sub_trj[t]) { auto p_itr = subtrj_matrix.back().sub_trj[t]->preconditions.find(var); if (p_itr != subtrj_matrix.back().sub_trj[t]->preconditions.cend()) { int causal_action_index = trajectories[t].causal_action_index(p_itr->second, p_itr->first); if (causal_action_index > 0) // If the arc exists and does not come from the start state states.push_back(trajectories[t].extract_state(causal_action_index + 1)); } } if (!states.empty()) { empty_states = false; Expression exp = Expression(states, var); if (!exp.empty()) { // Ignore conditions that are always true in every trajectory or the same as the overall termination if (exp == subtrj_matrix[0].termination) exp.clear(); else { unsigned t; for (t = 0; t < trajectories.size(); ++t) if (!trajectories[t].always_true(exp)) break; if (t == trajectories.size()) exp.clear(); } } if (exp.empty()) { var_precondition.clear(); return false; } var_precondition[var] = exp; } } return empty_states; } set CatScan::precondition_variables (const unsigned& index) const { set all_pre_vars; for (const auto& st : subtrj_matrix[index].sub_trj) if (st) { set pre_vars = extract_map_keys(st->preconditions); all_pre_vars.insert(pre_vars.begin(), pre_vars.end()); } return all_pre_vars; } void CatScan::process_admissibility (const set& precondition_variables) { // Admissibility restricted to the precondition variables vector> states; for (unsigned t = 0; t < trajectories.size(); ++t) if (subtrj_matrix.back().sub_trj[t] && !subtrj_matrix.back().sub_trj[t]->action_indices.empty()) states.push_back(trajectories[t].extract_state(*subtrj_matrix.back().sub_trj[t]->action_indices.begin())); if (!states.empty()) subtrj_matrix.back().admissibility = Expression(states, precondition_variables); } CatScan::Merger CatScan::process_merge (const vector& merge_column) const { Merger merge; for (const auto& mc : merge_column) { if (!mc.empty() && merge.empty()) { merge = mc; continue; } if (!mc.empty() && mc != merge) return Merger(); } return merge; } void CatScan::process_termination (vector& st_matrix, const vector>& merger) { for (unsigned c = 0; c < merger.size(); ++c) { Merger merge = process_merge(merger[c]); if (!merge.empty() && !merge.current() && merge.new_column) st_matrix[c].termination.conjunct(st_matrix[merge.column_index].termination); } } bool CatScan::add_column (Subtrj_Column& new_matrix_column, const vector& merge_column) { if (empty_column(new_matrix_column, merge_column)) return false; if (subsumed_column(new_matrix_column, merge_column)) return false; Merger merge = process_merge(merge_column); if (!merge.empty() && !merge.current() && !merge.new_column) { new_matrix_column.termination.conjunct(subtrj_matrix[merge.column_index].termination); new_matrix_column.column_indices.insert(subtrj_matrix[merge.column_index].column_indices.begin(), subtrj_matrix[merge.column_index].column_indices.end()); } for (unsigned t = 0; t < new_matrix_column.sub_trj.size(); ++t) { if (!merge_column[t].new_column && !merge_column[t].empty() && !merge_column[t].current()) new_matrix_column.sub_trj[t] = move(subtrj_matrix[merge_column[t].column_index].sub_trj[t]); else if (new_matrix_column.sub_trj[t]) { unsigned subtrj_column = check_subtrajectories(new_matrix_column.sub_trj[t]->action_indices, t); if (subtrj_column) subtrj_matrix[subtrj_column].sub_trj[t] = nullptr; } } subtrj_matrix.push_back(new_matrix_column); return true; } bool CatScan::empty_column (const Subtrj_Column& new_matrix_column, const vector& merge_column) const { bool is_empty_column = true; Merger merge; unsigned t; for (t = 0; t < merge_column.size(); ++t) { if (new_matrix_column.sub_trj[t]) is_empty_column = false; if (!merge_column[t].empty() && merge.empty()) { merge = merge_column[t]; continue; } if (!merge_column[t].empty() && merge_column[t] != merge) break; } if (t == merge_column.size() && is_empty_column && (merge.empty() || merge.current() || merge.new_column)) return true; return false; } bool CatScan::subsumed_column (const Subtrj_Column& new_matrix_column, const vector& merge_column) const { Merger merge; unsigned subtrj_column = 0; for (const auto& mc : merge_column) { if (!mc.empty() && merge.empty()) { merge = mc; if (merge.new_column) return false; continue; } if (!mc.empty() && mc != merge) return false; } if (!merge.empty() && !merge.current()) subtrj_column = merge.column_index; else { for (unsigned t = 0; t < new_matrix_column.sub_trj.size(); ++t) if (new_matrix_column.sub_trj[t]) { if (!subtrj_column) { subtrj_column = check_subtrajectories(new_matrix_column.sub_trj[t]->action_indices, t); if (!subtrj_column) return false; continue; } if (subtrj_column != check_subtrajectories(new_matrix_column.sub_trj[t]->action_indices, t)) return false; } } // Check strict subsumption for (unsigned t = 0; t < subtrj_matrix[subtrj_column].sub_trj.size(); ++t) if (subtrj_matrix[subtrj_column].sub_trj[t] && merge_column[t].empty() && !new_matrix_column.sub_trj[t]) return true; return false; } bool CatScan::unify (vector& st_matrix, const set& variables, vector>& merger) { // If two state variables coexist in any subset of a factoring (partition of the variables), then they are in the same subset of the final partition if (st_matrix.size() < 2) return true; // Determine global factoring vector> global_factors; vector>> local_factors; for (unsigned t = 0; t < trajectories.size(); ++t) { local_factors.push_back(vector>()); for (const auto& stm : st_matrix) { set factor; if (stm.sub_trj[t]) set_intersection(stm.sub_trj[t]->outgoing_variables.begin(), stm.sub_trj[t]->outgoing_variables.end(), variables.begin(), variables.end(), inserter(factor, factor.end())); local_factors.back().push_back(factor); } if (global_factors.empty()) global_factors = local_factors.back(); else { for (unsigned k = 0; k < st_matrix.size(); ++k) global_factors[k].insert(local_factors.back()[k].begin(), local_factors.back()[k].end()); } } // Collate global factors toward the end // Each column of the matrix is generated for a particular variable in the preconditions; // however, later columns might absorb subtrajectories associated with earlier columns for (unsigned gf = 0; gf < global_factors.size() - 1; ++gf) if (!global_factors[gf].empty()) { set new_factor(global_factors[gf]); global_factors[gf].clear(); unsigned fi = gf; for (unsigned f = gf + 1; f < global_factors.size(); ++f) { set result; set_intersection(new_factor.begin(), new_factor.end(), global_factors[f].begin(), global_factors[f].end(), inserter(result, result.end())); if (!result.empty()) { new_factor.insert(global_factors[f].begin(), global_factors[f].end()); global_factors[f].clear(); fi = f; } } global_factors[fi] = new_factor; } // Readjust the subtrajectories based on the global factors const vector matrix_reset(st_matrix); // Revert to original after failure const vector> merger_reset(merger); for (unsigned t = 0; t < trajectories.size(); ++t) { // Reconcile local factors with the global factoring by merging subtrajectories for (unsigned f = 0; f < global_factors.size(); ++f) if (!global_factors[f].empty()) { set action_indices; unsigned nonempty_subtrj = 0; set cleaner; for (unsigned lf = 0; lf <= f; ++lf) { if (!local_factors[t][lf].empty() && includes(global_factors[f].begin(), global_factors[f].end(), local_factors[t][lf].begin(), local_factors[t][lf].end())) { action_indices.insert(*st_matrix[lf].sub_trj[t]->action_indices.rbegin()); st_matrix[f].termination.conjunct(st_matrix[lf].termination); nonempty_subtrj = lf; cleaner.insert(lf); } } if (!action_indices.empty()) // Refactor when local factoring != global factoring { if (action_indices.size() == 1) { if (nonempty_subtrj != f) { st_matrix[f].sub_trj[t] = move(st_matrix[nonempty_subtrj].sub_trj[t]); merger[nonempty_subtrj][t].clear(); merger[f][t].set_current(); } } else { for (auto c_itr = cleaner.cbegin(); c_itr != cleaner.cend(); ++c_itr) { st_matrix[*c_itr].sub_trj[t] = nullptr; merger[*c_itr][t].clear(); } st_matrix[f].sub_trj[t] = trajectories[t].extract_subtrajectory(action_indices); if (st_matrix[f].sub_trj[t]) merger[f][t].set_current(); else // Unification failure { st_matrix = matrix_reset; merger = merger_reset; return false; } } } } } return true; } bool CatScan::merge (const set& subtrj_column_indices) { // Merge invalid subtrajectories together if (subtrj_column_indices.size() == 1) return false; const vector matrix_reset(subtrj_matrix); // Revert to original after failure subtrj_matrix.emplace_back(trajectories.size()); for (const auto& sci : subtrj_column_indices) { subtrj_matrix.back().termination.conjunct(subtrj_matrix[sci].termination); subtrj_matrix.back().column_indices.insert(subtrj_matrix[sci].column_indices.begin(), subtrj_matrix[sci].column_indices.end()); } for (unsigned t = 0; t < trajectories.size(); ++t) { unsigned nonempty_subtrj = 0; set new_action_indices; set cleaner; for (const auto& sci : subtrj_column_indices) if (subtrj_matrix[sci].sub_trj[t]) { new_action_indices.insert(*subtrj_matrix[sci].sub_trj[t]->action_indices.rbegin()); cleaner.insert(sci); nonempty_subtrj = sci; } if (!new_action_indices.empty()) { if (new_action_indices.size() == 1) subtrj_matrix.back().sub_trj[t] = move(subtrj_matrix[nonempty_subtrj].sub_trj[t]); else { for (const auto& c : cleaner) subtrj_matrix[c].sub_trj[t] = nullptr; subtrj_matrix.back().sub_trj[t] = trajectories[t].extract_subtrajectory(new_action_indices); if (!subtrj_matrix.back().sub_trj[t]) // Merge failure { subtrj_matrix = matrix_reset; return false; } // Absorb any subsumed subtrajectories /* for (unsigned k = 1; k < subtrj_matrix.size() - 1; ++k) { if (subtrj_matrix[k].sub_trj[t] && includes(subtrj_matrix.back().sub_trj[t]->action_indices.begin(), subtrj_matrix.back().sub_trj[t]->action_indices.end(), subtrj_matrix[k].sub_trj[t]->action_indices.begin(), subtrj_matrix[k].sub_trj[t]->action_indices.end())) subtrj_matrix[k].sub_trj[t] = nullptr; } */ } } } // Erase original columns unsigned erase_offset = 0; for (auto sci_itr = subtrj_column_indices.cbegin(); sci_itr != subtrj_column_indices.cend(); ++sci_itr, ++erase_offset) { subtrj_matrix.erase(subtrj_matrix.begin() + (*sci_itr - erase_offset)); // Adjust subtrj column indices for (unsigned c = 1; c < subtrj_matrix.size(); ++c) for (auto ci_itr = subtrj_matrix[c].column_indices.cbegin(); ci_itr != subtrj_matrix[c].column_indices.cend(); ) { if (*ci_itr > *sci_itr - erase_offset) { subtrj_matrix[c].column_indices.insert(*ci_itr - 1); subtrj_matrix[c].column_indices.erase(ci_itr++); } else ++ci_itr; } } return true; } set CatScan::abstraction (const unsigned& index, const ActionModel& action_model) const { set actions; set goal_variables; for (unsigned t = 0; t < subtrj_matrix[index].sub_trj.size(); ++t) if (subtrj_matrix[index].sub_trj[t]) { // Actions set trajectory_actions = trajectories[t].extract_actions(subtrj_matrix[index].sub_trj[t]->action_indices); actions.insert(trajectory_actions.begin(), trajectory_actions.end()); // Goal variables goal_variables.insert(subtrj_matrix[index].sub_trj[t]->outgoing_variables.begin(), subtrj_matrix[index].sub_trj[t]->outgoing_variables.end()); } return action_model.closure(actions, goal_variables); } vector CatScan::subtrajectories (const unsigned& index) const { vector subtrajectories; for (unsigned t = 0; t < trajectories.size(); ++t) if (subtrj_matrix[index].sub_trj[t] && !subtrj_matrix[index].sub_trj[t]->action_indices.empty()) subtrajectories.emplace_back(trajectories[t], subtrj_matrix[index].sub_trj[t]->action_indices); return subtrajectories; } unsigned CatScan::check_subtrajectories (const set& action_indices, const int& t) const { // Check if the set of causal actions has been expanded before for (unsigned k = 1; k < subtrj_matrix.size(); ++k) if (subtrj_matrix[k].sub_trj[t] && includes(subtrj_matrix[k].sub_trj[t]->action_indices.begin(), subtrj_matrix[k].sub_trj[t]->action_indices.end(), action_indices.begin(), action_indices.end())) return k; return 0; // Not found } int CatScan::check_subtrajectories (const vector& st_matrix, const int& action_index, const int& t) const { // Check if the set of causal actions has been expanded before for (unsigned k = 0; k < st_matrix.size() - 1; ++k) if (st_matrix[k].sub_trj[t] && st_matrix[k].sub_trj[t]->action_indices.find(action_index) != st_matrix[k].sub_trj[t]->action_indices.end()) return k; return -1; // Not found } void CatScan::precedence_graph_builder () { precedence_graph.resize(subtrj_matrix.size()); for (unsigned c = 0; c < subtrj_matrix.size(); ++c) for (const auto& ci : subtrj_matrix[c].column_indices) precedence_graph.set_edge(c, ci, 0); // 0 = causal edge if (!precedence_graph.is_acyclic()) throw HierException(__FILE__, __LINE__, "Precedence graph is not acyclic."); precedence_graph_postprocessor(); //precedence_graph_temporal_links(); //precedence_graph.print_gv("precedence_graph.gv"); //if (system("dot -Tpdf -o precedence_graph.pdf precedence_graph.gv")) // cerr << "(" + to_string(__FILE__) + ":" + to_string(__LINE__) + ")\n"; } void CatScan::precedence_graph_postprocessor () { precedence_graph.transitive_reduction(0); // If the root task has a single parent, then relocate grandparents to root (and leave newly orphaned parent) set root_parents = precedence_graph.parents(0, 0); if (root_parents.size() == 1) { unsigned root_parent = *root_parents.begin(); set new_parents = precedence_graph.parents(root_parent, 0); for (const auto& np : new_parents) { precedence_graph.set_edge(np, 0, 0); precedence_graph.remove_edge(np, root_parent, 0); } subtrj_matrix[root_parent].termination.clear(); // For the hierarchy builder to know that the subtrajectories are associated with the parent } } void CatScan::precedence_graph_temporal_links () { // Protect causal links among unordered precondition tasks for (unsigned i = 0; i < precedence_graph.num_vertices(); ++i) { set precondition_tasks = precedence_graph.parents(i); for (auto pt_itr1 = precondition_tasks.cbegin(); pt_itr1 != precondition_tasks.cend(); ++pt_itr1) { auto pt_itr2 = pt_itr1; ++pt_itr2; for ( ; pt_itr2 != precondition_tasks.cend(); ++pt_itr2) if (*pt_itr1 != *pt_itr2 && precedence_graph.shortest_path(*pt_itr1, *pt_itr2).empty() && precedence_graph.shortest_path(*pt_itr2, *pt_itr1).empty()) { // Add temporal edge if (causal_threat(*pt_itr1, *pt_itr2)) precedence_graph.set_edge(*pt_itr1, *pt_itr2, 1); if (causal_threat(*pt_itr2, *pt_itr1)) precedence_graph.set_edge(*pt_itr2, *pt_itr1, 1); } } } } bool CatScan::causal_threat (const unsigned& task1, const unsigned& task2) const { // Check if task1's preconditions are clobbered by task2 set task1_vars = subtrj_matrix[task1].admissibility.lhs_variables(); set task2_vars = subtrj_matrix[task2].termination.lhs_variables(); set result; set_intersection(task1_vars.begin(), task1_vars.end(), task2_vars.begin(), task2_vars.end(), inserter(result, result.end())); return !result.empty(); } void CatScan::cleanup () { // Remove any fully empty columns for (unsigned k = 1; k < subtrj_matrix.size(); ) { unsigned t; for (t = 0; t < subtrj_matrix[k].sub_trj.size(); ++t) if (subtrj_matrix[k].sub_trj[t]) break; if (t == subtrj_matrix[k].sub_trj.size()) { subtrj_matrix.erase(subtrj_matrix.begin() + k); // Adjust subtrj column indices for (unsigned c = 1; c < subtrj_matrix.size(); ++c) for (auto ci_itr = subtrj_matrix[c].column_indices.cbegin(); ci_itr != subtrj_matrix[c].column_indices.cend(); ) { if (*ci_itr >= k) { if (*ci_itr > k) subtrj_matrix[c].column_indices.insert(*ci_itr - 1); subtrj_matrix[c].column_indices.erase(ci_itr++); } else ++ci_itr; } } else ++k; } } hierlearning/hiergen/model.cc0000664000175000017500000004137612205065452016451 0ustar nevilleneville /************************************************* MODEL INTERFACE Neville Mehta **************************************************/ #include #include #include #include "model.h" #if defined(NDEBUG) const bool display = false; #else const bool display = true; #endif double pruning_confidence = 0; // Closer to zero = more pruning (Weka complains below 1e-6 and above 1); 0 = unpruned ModelTree::ModelTree (const string& filename, const State& state) : leaf(0) { ifstream model_file(filename.c_str()); if (!model_file.is_open()) throw HierException(__FILE__, __LINE__, "Model file " + filename + " does not exist."); list model_file_lines; while (!model_file.eof()) { string line; getline_os(model_file, line); if (!line.empty()) model_file_lines.push_back(line); } read_file(model_file_lines, state, 0); model_file.close(); } ModelTree::ModelTree (list& model_file_lines, const State& state, const int& level) : leaf(0) { read_file(model_file_lines, state, level); } void ModelTree::read_file (list& model_file_lines, const State& state, const int& level) { while (!model_file_lines.empty()) { string line = model_file_lines.front(); model_file_lines.pop_front(); // Determine recursion level of current line int count = 0; size_t index = line.find_first_of("|"); while (index != string::npos) { ++count; index = line.find_first_of("|", index + 1); } if (count < level) { model_file_lines.push_front(line); break; } index = line.find_first_of(":"); if (index == string::npos) // No leaf value exists choices.emplace_back(Expression(line.substr(line.find_first_not_of(" |")), state), ModelTree(model_file_lines, state, level + 1)); else { size_t start = line.find_first_not_of(" |"); if (index - start) choices.emplace_back(Expression(line.substr(start, index - start), state), ModelTree(from_string(line.substr(index + 2)))); else leaf = from_string(line.substr(index + 2)); } } } int ModelTree::true_choice (const State& state) const { for (unsigned i = 0; i < choices.size(); ++i) if (choices[i].first.evaluate(state)) return i; throw HierException(__FILE__, __LINE__, "No satisfying path in action model tree."); } int ModelTree::true_choice (const map& variable_value) const { for (unsigned i = 0; i < choices.size(); ++i) if (choices[i].first.evaluate(variable_value)) return i; throw HierException(__FILE__, __LINE__, "No satisfying path in action model tree."); } int ModelTree::get_value (const State& state) const { if (is_leaf()) return leaf; return choices[true_choice(state)].second.get_value(state); } int ModelTree::get_value (const map& variable_value) const { if (is_leaf()) return leaf; return choices[true_choice(variable_value)].second.get_value(variable_value); } set ModelTree::leaf_values () const { if (is_leaf()) return make_set(1, leaf); set all_leaf_values; for (const auto& choice : choices) { set child_leaf_values = choice.second.leaf_values(); all_leaf_values.insert(child_leaf_values.begin(), child_leaf_values.end()); } return all_leaf_values; } bool ModelTree::is_persistent (const int& variable) const { // Presumes that every split is based on the domain values of a single variable, e.g., v0 = 0, v0 = 1, v0 = 2, ... if (is_leaf()) return false; for (const auto& choice : choices) { set lhs_variable = choice.first.variables(); if (lhs_variable.size() > 1 || *lhs_variable.begin() != variable) return false; set leaf_values = choice.second.leaf_values(); if (leaf_values.size() > 1 || *leaf_values.begin() != choice.first.rhs_value()) return false; } return true; } set ModelTree::checked_variables () const { if (is_leaf() || leaf_values().size() == 1) // Checking for redundant subtrees return set(); set variables; for (const auto& choice : choices) { set vars = choice.first.variables(); variables.insert(vars.begin(), vars.end()); vars = choice.second.checked_variables(); variables.insert(vars.begin(), vars.end()); } return variables; } set ModelTree::context_checked_variables (const map& variable_value) const { if (is_leaf() || leaf_values().size() == 1) // Checking for redundant subtrees return set(); int tc = true_choice(variable_value); set abstraction = choices[tc].first.variables(); set child_abstraction = choices[tc].second.context_checked_variables(variable_value); abstraction.insert(child_abstraction.begin(), child_abstraction.end()); return abstraction; } string ModelTree::print_gv_cluster (const string& node_prefix, int& id) const { string model_str; if (is_leaf()) model_str += node_prefix + to_string(id) + " [label=\"" + to_string(leaf) + "\"]\n"; else { // Values on edges only if all are equality tests for the same lhs variable bool value_edge_label = false; Expression internal_node = choices.front().first; if (internal_node.equality()) { value_edge_label = true; for (unsigned i = 1; i < choices.size(); ++i) if (choices[i].first.lhs_expression() != choices.front().first.lhs_expression()) { value_edge_label = false; break; } } int parent_id = id; if (value_edge_label) { model_str += node_prefix + to_string(parent_id) + " [label=<" + choices.front().first.lhs_expression().print_html() + ">]\n"; for (const auto& choice : choices) { model_str += node_prefix + to_string(parent_id) + " -> " + node_prefix + to_string(++id) + " [label=<" + choice.first.rhs_expression().print_html() + ">]\n"; model_str += choice.second.print_gv_cluster(node_prefix, id); } } else { model_str += node_prefix + to_string(parent_id) + " [label=\"\"]\n"; for (const auto& choice : choices) { model_str += node_prefix + to_string(parent_id) + " -> " + node_prefix + to_string(++id) + " [label=<" + choice.first.print_html() + ">]\n"; model_str += choice.second.print_gv_cluster(node_prefix, id); } } } return model_str; } //------------------------------------------------------------------------------------------------- ActionModel::ActionModel (const string& trajectory_filename, const string& model_directory, const MDP& mdp) { string action_model_directory; if (!model_directory.empty()) action_model_directory = model_directory; else { size_t i = trajectory_filename.find_last_of("\\/"); action_model_directory = "output/" + mdp.name() + "/models_" + trajectory_filename.substr(i + 1, trajectory_filename.find_first_of(".") - i - 1); create_directory(action_model_directory); if (system(("python model_builder.py " + trajectory_filename + " " + action_model_directory + " " + to_string(pruning_confidence) + " " + (display ? "True" : "False")).c_str())) throw HierException(__FILE__, __LINE__, "There was a problem building action models."); } read(action_model_directory, mdp); if (display && model_directory.empty()) print_gv(action_model_directory, mdp); } ActionModel::ActionModel (const vector>>& input_trajectories, const string& model_directory, const unsigned& run_index, const MDP& mdp) { string action_model_directory; if (!model_directory.empty()) action_model_directory = model_directory; else { action_model_directory = "output/" + mdp.name() + "/models_internal_trajectory_" + to_string(run_index); create_directory(action_model_directory); string trajectory_filename = "output/" + mdp.name() + "/internal_trajectory_" + to_string(run_index) + ".out"; ofstream trajectory_file(trajectory_filename.c_str()); if (!trajectory_file.is_open()) throw HierException(__FILE__, __LINE__, "Unable to write trajectories to file."); // Header information vector mdp_actions = mdp.actions(); for (unsigned a = 0; a < mdp_actions.size(); ++a) { if (a > 0) trajectory_file << " "; trajectory_file << mdp.print_action(mdp_actions[a]); } trajectory_file << endl; for (unsigned a = 0; a < mdp_actions.size(); ++a) { if (a > 0) trajectory_file << " "; trajectory_file << mdp_actions[a]; } trajectory_file << endl; vector state_variables = mdp.state().variables(); for (auto v = state_variables.cbegin(); v != state_variables.cend(); ++v) { if (v != state_variables.begin()) trajectory_file << " "; trajectory_file << mdp.state().variable_name(*v); } trajectory_file << endl; for (auto v = state_variables.cbegin(); v != state_variables.cend(); ++v) { if (v != state_variables.begin()) trajectory_file << " "; trajectory_file << mdp.state().variable_size(*v); } trajectory_file << "\n#\n"; for (const auto& trajectory : input_trajectories) if (!trajectory.empty()) { for (auto sar = trajectory.cbegin(); sar != trajectory.cend(); ++sar) { if (sar + 1 != trajectory.end()) trajectory_file << (*sar)->state->print() << endl << (*sar)->action[0] << endl << (*sar)->reward[0] << endl; else trajectory_file << (*sar)->state->print() << endl; } trajectory_file << "#\n"; } trajectory_file.close(); if (system(("python model_builder.py " + trajectory_filename + " " + action_model_directory + " " + to_string(pruning_confidence) + " " + (display ? "True" : "False")).c_str())) throw HierException(__FILE__, __LINE__, "There was a problem building action models."); remove(trajectory_filename.c_str()); } read(action_model_directory, mdp); if (model_directory.empty()) { if (display) print_gv(action_model_directory, mdp); else remove_directory(action_model_directory.c_str()); } } void ActionModel::read (const string& directory, const MDP& mdp) { vector mdp_actions = mdp.actions(); vector state_variables = mdp.state().variables(); if (display) cout << "Reading models ...\n"; for (const auto& mdp_action : mdp_actions) { string action_name = mdp.print_action(mdp_action); if (display) cout << action_name << endl; for (const auto& var : state_variables) { string filename = action_name + "__" + mdp.state().variable_name(var) + ".model"; _model[mdp_action].transition[var] = ModelTree(directory + "/" + filename, mdp.state()); } string filename = action_name + "__reward.model"; _model[mdp_action].reward = ModelTree(directory + "/" + filename, mdp.state()); } if (display) cout << "Done.\n"; compute_checked_variables(); } void ActionModel::compute_checked_variables () { for (auto& am : _model) { // Transition for (const auto& var : am.second.transition) if (!var.second.is_persistent(var.first)) { set checked_per_variable = var.second.checked_variables(); am.second.transition_checked_variables.insert(checked_per_variable.begin(), checked_per_variable.end()); } // Reward am.second.reward_checked_variables = am.second.reward.checked_variables(); am.second.reward_checked_variables_depth_2 = closure_next_stage(am, am.second.reward_checked_variables); } } set ActionModel::closure_next_stage (const pair& am, const set& closure_variables) const { set next_closure_variables = closure_variables; for (const auto& var : closure_variables) { auto v = am.second.transition.find(var); if (!v->second.is_persistent(v->first)) { set checked_per_variable = v->second.checked_variables(); next_closure_variables.insert(checked_per_variable.begin(), checked_per_variable.end()); } } return next_closure_variables; } set ActionModel::closure (const set& actions, const set& variables) const { set closure_variables = variables; // Collect reward variables from all actions for (const auto& action : actions) { auto am_itr = _model.find(action); if (am_itr == _model.end()) throw HierException(__FILE__, __LINE__, "No model exists for the provided action."); closure_variables.insert(am_itr->second.reward_checked_variables.begin(), am_itr->second.reward_checked_variables.end()); } while (true) { set next_stage_closure_variables = closure_variables; for (const auto& action : actions) { set next_stage_closure_variables_per_action = closure_next_stage(*_model.find(action), closure_variables); next_stage_closure_variables.insert(next_stage_closure_variables_per_action.begin(), next_stage_closure_variables_per_action.end()); } if (next_stage_closure_variables == closure_variables) break; closure_variables = next_stage_closure_variables; } return closure_variables; } set ActionModel::reward_variables (const int& action) const { auto am_itr = _model.find(action); if (am_itr == _model.end()) throw HierException(__FILE__, __LINE__, "No model exists for the provided action."); return am_itr->second.reward_checked_variables_depth_2; } set ActionModel::checked_variables (const int& action) const { auto am_itr = _model.find(action); if (am_itr == _model.end()) throw HierException(__FILE__, __LINE__, "No model exists for the provided action."); return am_itr->second.transition_checked_variables; } set ActionModel::context_checked_variables (const int& action, const map& variable_value) const { auto am_itr = _model.find(action); if (am_itr == _model.end()) throw HierException(__FILE__, __LINE__, "No model exists for the provided action."); set checked_variables; // Transition for (const auto& v : am_itr->second.transition) if (!v.second.is_persistent(v.first)) { set checked_per_variable = v.second.context_checked_variables(variable_value); checked_variables.insert(checked_per_variable.begin(), checked_per_variable.end()); } // Reward set reward_variables = am_itr->second.reward.context_checked_variables(variable_value); checked_variables.insert(reward_variables.begin(), reward_variables.end()); return checked_variables; } set ActionModel::context_changed_variables (const int& action, const map& variable_value) const { // Doesn't apply to stochastic outcomes auto am_itr = _model.find(action); if (am_itr == _model.end()) throw HierException(__FILE__, __LINE__, "No model exists for the provided action."); set changed_variables; for (const auto& v : am_itr->second.transition) if (variable_value.find(v.first)->second != v.second.get_value(variable_value)) // Variable's value changes according to model changed_variables.insert(v.first); return changed_variables; } void ActionModel::print_gv (const string& directory, const MDP& mdp) const { if (display) cout << "Printing models ...\n"; for (const auto& am : _model) { string gv_filename = directory + "/" + mdp.print_action(am.first); if (display) cout << gv_filename << ".pdf\n"; ofstream model_file((gv_filename + ".gv").c_str()); if (!model_file.is_open()) throw HierException(__FILE__, __LINE__, "Unable to print action model."); model_file << "digraph {\nedge [arrowhead=empty]\n\n"; for (const auto& m : am.second.transition) { model_file << "subgraph cluster_" << mdp.state().variable_name(m.first) << " {\nlabel=\"" << mdp.state().variable_name(m.first) << "\"\n"; model_file << m.second.print_gv_cluster(mdp.state().variable_name(m.first) + "_n"); model_file << "}\n\n"; } model_file << "subgraph cluster_reward {\nlabel=\"reward\"\n"; model_file << am.second.reward.print_gv_cluster("reward_n"); model_file << "}\n}\n"; model_file.close(); if (system(("dot -Tpdf -o " + gv_filename + ".pdf " + gv_filename + ".gv").c_str())) cerr << "(" + to_string(__FILE__) + ":" + to_string(__LINE__) + ")\n"; } if (display) cout << "Done.\n"; } void ActionModel::print_context_precondition (const MDP& mdp) const { ofstream precondition_file("precondition.txt"); if (!precondition_file.is_open()) throw HierException(__FILE__, __LINE__, "Unable to print precondition file."); map variable_size; vector state_variables = mdp.state().variables(); for (const auto& var : state_variables) variable_size[var] = mdp.state().variable_size(var); vector mdp_actions = mdp.actions(); for (unsigned a = 0; a < mdp_actions.size(); ++a) { map state; for (const auto& var : state_variables) state[var] = 0; precondition_file << mdp.print_action(mdp_actions[a]) << endl; do { set precondition; try { precondition = context_checked_variables(a, state); } catch (...) // ModelTree throws an exception when no path is true {} if (!precondition.empty()) { precondition_file << state << " : "; for (const auto& p : precondition) precondition_file << p << " "; precondition_file << endl; } bool carry = true; for (auto v = state.begin(); v != state.end() && carry; ++v) { ++v->second; map::const_iterator u = v; if (v->second == variable_size[v->first] && ++u != state.end()) { v->second = 0; carry = true; } else carry = false; } } while (state.rbegin()->second < (int)variable_size[state.rbegin()->first]); precondition_file << endl << endl; } precondition_file.close(); } hierlearning/hiergen/catscan.h0000664000175000017500000000773412205065452016627 0ustar nevilleneville /************************************************* CAUSALLY ANNOTATED TRAJECTORY SCANNER Neville Mehta **************************************************/ #pragma once #include #include #include #include "../lib/graph.h" #include "trajectory.h" class CatScan { struct Subtrj_Column { vector> sub_trj; // One for each trajectory Expression admissibility; Expression termination; set column_indices; // Column indices for which current column is a precondition Subtrj_Column (const unsigned& num_trajectories) : sub_trj(vector>(num_trajectories)) {} Subtrj_Column (const unsigned& num_trajectories, const Expression& termination) : sub_trj(vector>(num_trajectories)), termination(termination) {} Subtrj_Column (const unsigned& num_trajectories, const Expression& termination, const unsigned& column_index) : sub_trj(vector>(num_trajectories)), termination(termination), column_indices(make_set(1, column_index)) {} }; vector subtrj_matrix; const vector& trajectories; graph precedence_graph; struct Merger { int column_index; bool new_column; Merger (const int& column_index = -2, const bool& new_column = false) : column_index(column_index), new_column(new_column) {} bool empty () const { return column_index == -2; } void clear () { column_index = -2; } bool current () const { return column_index == -1; } void set_current () { column_index = -1; } void set (const int& col_index, const bool& new_col) { column_index = col_index; new_column = new_col; } bool operator == (const Merger& merge) const { return column_index == merge.column_index && new_column == merge.new_column; } bool operator != (const Merger& merge) const { return !(*this == merge); } }; bool process(); bool preconditions(map& var_precondition) const; set precondition_variables(const unsigned& index) const; void process_admissibility(const set& precondition_variables); Merger process_merge(const vector& merge_column) const; void process_termination(vector& st_matrix, const vector>& merger); bool add_column(Subtrj_Column& new_matrix_column, const vector& merge_column); bool empty_column(const Subtrj_Column& new_matrix_column, const vector& merge_column) const; bool subsumed_column(const Subtrj_Column& new_matrix_column, const vector& merge_column) const; bool unify(vector& st_matrix, const set& variables, vector>& merger); bool merge(const set& subtrj_column_indices); unsigned check_subtrajectories(const set& action_indices, const int& t) const; unsigned check_subtrajectories(const int& action_index, const int& t) const { return check_subtrajectories(make_set(1, action_index), t); } int check_subtrajectories(const vector& st_matrix, const int& action_index, const int& t) const; void precedence_graph_builder(); void precedence_graph_postprocessor(); void precedence_graph_temporal_links(); bool causal_threat(const unsigned& task1, const unsigned& task2) const; void cleanup(); public: CatScan (const vector& trajectories, const Expression& termination); bool decomposed () const { return subtrj_matrix.size() > 2; } unsigned num_subtasks () const { return subtrj_matrix.size() - 1; } set subtask_indices (const unsigned& index) const { return precedence_graph.parents(index, 0); } // Precondition subtrajectories are child tasks set abstraction(const unsigned& index, const ActionModel& action_model) const; vector subtrajectories(const unsigned& index) const; Expression admissibility (const unsigned& index) const { return subtrj_matrix[index].admissibility; } Expression termination (const unsigned& index) const { return subtrj_matrix[index].termination; } ~CatScan () {} }; hierlearning/hiergen/hierarchy.cc0000664000175000017500000002321212205065452017314 0ustar nevilleneville /************************************************* HIERARCHICAL ANALYSIS Neville Mehta **************************************************/ #include #include #include "../lib/common.h" #include "../learner/hier/generic/primitive_task.h" #include "hierarchy.h" #if defined(NDEBUG) const bool display = false; #else const bool display = true; #endif Hierarchy::Hierarchy (const string& trajectory_filename, const string& model_directory, const MDP& mdp) : _is_root(true), _primitive_action(0) { ifstream tf(trajectory_filename.c_str()); if (!tf.is_open()) throw HierException(__FILE__, __LINE__, "Unable to open " + trajectory_filename + "."); ActionModel action_model(trajectory_filename, model_directory, mdp); // Trajectory file header string header; getline_os(tf, header); // Actions getline_os(tf, header); // Action indices getline_os(tf, header); // Variables vector vars = tokenize(header, " "); if (vars.size() != mdp.state().variables().size()) throw HierException(__FILE__, __LINE__, "Trajectory variables do not match that of the MDP."); while (header != "#") getline_os(tf, header); if (display) cout << "Annotating trajectories ..."; vector trajectories; while (true) { char c = 0; while (tf >> c && c == '#'); // Gobble up empty trajectories if (tf.eof()) break; tf.putback(c); if (display) cout << " " << trajectories.size() + 1; trajectories.emplace_back(tf, mdp, action_model); if (trajectories.size() > 1 && extract_map_keys(trajectories.back().extract_goal_state()) != extract_map_keys(trajectories[trajectories.size() - 2].extract_goal_state())) throw HierException(__FILE__, __LINE__, "Malformed state detected."); } tf.close(); if (display) cout << ".\n"; if (trajectories.empty()) throw HierException(__FILE__, __LINE__, "No input trajectories in " + trajectory_filename + "."); if (display) print_cats("output/" + mdp.name() + "/cats", trajectories, mdp); termination_condition(trajectories); state_abstraction(trajectories, action_model, _termination.variables()); hierarchicalize(trajectories, action_model); } Hierarchy::Hierarchy (const vector>>& input_trajectories, const string& model_directory, const unsigned& run_index, const MDP& mdp) : _is_root(true), _primitive_action(-1) { if (input_trajectories.empty()) throw HierException(__FILE__, __LINE__, "No input trajectories."); ActionModel action_model(input_trajectories, model_directory, run_index, mdp); if (display) cout << "Annotating trajectories ..."; vector trajectories; for (const auto& trajectory : input_trajectories) if (!trajectory.empty()) { if (display) cout << " " << trajectories.size() + 1; trajectories.emplace_back(trajectory, action_model); if (trajectories.size() > 1 && extract_map_keys(trajectories.back().extract_goal_state()) != extract_map_keys(trajectories[trajectories.size() - 2].extract_goal_state())) throw HierException(__FILE__, __LINE__, "Malformed state detected."); } if (display) cout << "done.\n"; if (display) print_cats("output/" + mdp.name() + "/cats_internal", trajectories, mdp); termination_condition(trajectories); state_abstraction(trajectories, action_model, _termination.variables()); hierarchicalize(trajectories, action_model); } void Hierarchy::state_abstraction (const vector& trajectories, const ActionModel& action_model, const set& goal_variables) { set actions; for (const auto& trajectory : trajectories) { set trajectory_actions = trajectory.extract_actions(); actions.insert(trajectory_actions.begin(), trajectory_actions.end()); } _abstraction = action_model.closure(actions, goal_variables); } void Hierarchy::termination_condition (const vector& trajectories) { vector> final_states; for (const auto& trajectory : trajectories) final_states.push_back(trajectory.extract_goal_state()); _termination = Expression(final_states, trajectories[0].goal_variables()); } void Hierarchy::hierarchicalize (const vector& trajectories, const ActionModel& action_model) { if (!_termination.empty()) { CatScan catscan(trajectories, _termination); if (catscan.decomposed()) { _subtasks = incorporate_subtasks(catscan, action_model); return; } } // No decomposition; include primitive actions as children map> action_abstractions = extract_action_abstractions(trajectories, action_model); for (const auto& aa : action_abstractions) { _subtasks.push_back(new Hierarchy(aa.first, aa.second)); _subtask_library.emplace_back(0, _subtasks.back()); } } vector Hierarchy::incorporate_subtasks (const CatScan& catscan, const ActionModel& action_model, const unsigned& index, const bool& precondition_ordering, const bool& admissibility_wrapper) { vector subtasks; if (precondition_ordering) { set child_task_indices = catscan.subtask_indices(index); for (const auto& child_index : child_task_indices) { vector>::const_iterator sl_itr = find_if(_subtask_library.begin(), _subtask_library.end(), [&](const pair& p){ return p.first == child_index; }); if (sl_itr != _subtask_library.end()) subtasks.push_back(sl_itr->second); else { vector child_tasks = incorporate_subtasks(catscan, action_model, child_index, precondition_ordering, admissibility_wrapper); Hierarchy* hierarchy = new Hierarchy(set(), Expression(), catscan.termination(child_index), catscan.subtrajectories(child_index), action_model); // Hierarchy* hierarchy = new Hierarchy(set(), Expression(), Expression(), catscan.subtrajectories(*c_itr), action_model); // To incorporate the primitive actions without further decomposition _subtask_library.emplace_back(0, hierarchy); if (admissibility_wrapper && !child_tasks.empty() && !catscan.admissibility(child_index).empty() && hierarchy->_subtasks.size() == 1 && hierarchy->_subtasks.front()->_subtasks.empty()) { // Only wrap single primitive action with admissibility hierarchy->_abstraction = hierarchy->_subtasks.front()->_abstraction; hierarchy->_admissibility = catscan.admissibility(child_index); child_tasks.push_back(hierarchy); } else child_tasks.insert(child_tasks.end(), hierarchy->_subtasks.begin(), hierarchy->_subtasks.end()); if (catscan.termination(child_index).empty() && (!admissibility_wrapper || catscan.admissibility(child_index).empty() || child_tasks.size() != 1 || !child_tasks.front()->_subtasks.empty())) subtasks.insert(subtasks.end(), child_tasks.begin(), child_tasks.end()); else { if (admissibility_wrapper && !catscan.admissibility(child_index).empty() && child_tasks.size() == 1 && child_tasks.front()->_subtasks.empty()) subtasks.push_back(new Hierarchy(child_tasks.front()->_abstraction, catscan.admissibility(child_index), Expression(), child_tasks)); else subtasks.push_back(new Hierarchy(catscan.abstraction(child_index, action_model), Expression(), catscan.termination(child_index), child_tasks)); _subtask_library.emplace_back(child_index, subtasks.back()); } } } } else for (unsigned i = 1; i <= catscan.num_subtasks(); ++i) { Hierarchy* hierarchy = new Hierarchy(catscan.abstraction(i, action_model), catscan.admissibility(i), catscan.termination(i), catscan.subtrajectories(i), action_model); _subtask_library.push_back(make_pair(i, hierarchy)); if (hierarchy->_subtasks.size() == 1 && hierarchy->_subtasks.front()->_subtasks.empty()) hierarchy->_abstraction = hierarchy->_subtasks.front()->_abstraction; subtasks.push_back(hierarchy); } return subtasks; } map> Hierarchy::extract_action_abstractions (const vector& trajectories, const ActionModel& action_model) const { map> action_abstractions; for (const auto& trajectory : trajectories) { set actions = trajectory.extract_actions(); for (const auto& action : actions) if (action_abstractions.find(action) == action_abstractions.end()) action_abstractions[action] = action_model.reward_variables(action); } return action_abstractions; } bool Hierarchy::combinable_tasks (vector& subtasks, Task* target_task) const { // Combine if the tasks are not connected or target is an immediate child for (const auto& subtask : subtasks) if (connected_tasks(subtask.link, target_task) && subtask.link != target_task) return false; return true; } bool Hierarchy::connected_tasks (Task* source_task, Task* target_task) const { if (source_task == target_task) return true; if (!source_task->primitive()) { CompositeTask* comp_task = dynamic_cast(source_task); for (unsigned t = 0; t < comp_task->num_subtasks(); ++t) if (connected_tasks(comp_task->get_subtask(t), target_task)) return true; } return false; } string Hierarchy::abstraction_string (const set& abstraction_variables, const MDP& mdp) const { // Stringifying the abstraction string abstraction_str; for (const auto& var : abstraction_variables) abstraction_str += mdp.state().variable_name(var) + ", "; if (!abstraction_str.empty()) abstraction_str.erase(abstraction_str.size() - 2); return abstraction_str; } void Hierarchy::print_cats (const string& directory, const vector& trajectories, const MDP& mdp) const { if (display) cout << "Printing cats ..."; create_directory(directory); for (unsigned t = 0; t < trajectories.size(); ++t) { if (display) cout << " " << t + 1; trajectories[t].print_gv(directory + "/cat_" + to_string(t + 1), mdp); } if (display) cout << ".\n"; } Hierarchy::~Hierarchy () { for (const auto& sl : _subtask_library) delete sl.second; } hierlearning/hiergen/trajectory.cc0000664000175000017500000003651212213030152017517 0ustar nevilleneville /************************************************* CAUSALLY ANNOTATED TRAJECTORY Neville Mehta **************************************************/ #include #include #include "../lib/common.h" #include "trajectory.h" Trajectory::Trajectory (istream& in, const MDP& mdp, const ActionModel& action_model) { if (!in.eof()) { _trajectory.emplace_back(map(), Start_Action); // Insert Start action unique_ptr mdp_state = mdp.state().copy(); char c = 0; while (!in.eof() && c != '#') { int action = End_Action; double reward = 0.0; mdp_state->read(in); if (!in.eof()) { in >> c; // Check for trajectory separator symbol if (c != '#') { in.putback(c); if (!(in >> action) || !(in >> reward)) throw HierException(__FILE__, __LINE__, "Bad trajectory format."); } } map state = mdp_state->variables_mapper(); // Remove loops through the state space for (auto trj_itr = _trajectory.rbegin(); trj_itr != _trajectory.rend(); ++trj_itr) if (trj_itr->state == state) { _trajectory.erase(trj_itr.base() - 1, _trajectory.end()); break; } _trajectory.emplace_back(state, action, reward); } if (_trajectory.size() >= 3) { causal_annotation(action_model, mdp.state().goal_variables(), mdp.state().transfer_variables()); cleanup(); return; } } throw HierException(__FILE__, __LINE__, "Unable to read the trajectory."); } Trajectory::Trajectory (const vector>& sample_trajectory, const ActionModel& action_model) { if (!sample_trajectory.empty()) { _trajectory.emplace_back(map(), Start_Action); // Insert Start action for (auto sar_itr = sample_trajectory.cbegin(); sar_itr != sample_trajectory.cend(); ++sar_itr) { map state = (*sar_itr)->state->variables_mapper(); // Removing redundant loops through the state space for (auto trj_itr = _trajectory.rbegin(); trj_itr != _trajectory.rend(); ++trj_itr) if (trj_itr->state == state) { _trajectory.erase(trj_itr.base() - 1, _trajectory.end()); break; } if (sar_itr + 1 != sample_trajectory.end()) _trajectory.emplace_back(state, (*sar_itr)->action.front(), (*sar_itr)->reward[0]); else _trajectory.emplace_back(state, End_Action); } if (_trajectory.size() >= 3) { causal_annotation(action_model, (*sample_trajectory.begin())->state->goal_variables(), (*sample_trajectory.begin())->state->transfer_variables()); cleanup(); return; } } throw HierException(__FILE__, __LINE__, "Input trajectory is empty."); } Trajectory::Trajectory (const Trajectory& trajectory, const set& action_indices) { // Adding Start & End actions, and reconfiguring causal arcs if (trajectory._trajectory.empty() || action_indices.empty()) return; map new_indices; _trajectory.emplace_back(map(), Start_Action); for (const auto& action_index : action_indices) { _trajectory.push_back(trajectory._trajectory[action_index]); new_indices[action_index] = _trajectory.size() - 1; } _trajectory.emplace_back(trajectory._trajectory[*action_indices.rbegin() + 1].state, End_Action); for (unsigned node_idx = 1; node_idx < _trajectory.size() - 1; ++node_idx) { // Reconfigure incoming arcs for real actions for (auto& arc_in : _trajectory[node_idx].causal_arcs.incoming) { if (new_indices.find(arc_in.second) == new_indices.end()) // Arc points from without current sub-trajectory { arc_in.second = 0; // Arc tail at Start _trajectory.front().causal_arcs.outgoing[arc_in.first].insert(node_idx); } else arc_in.second = new_indices[arc_in.second]; } // Reconfigure outgoing arcs for real actions for (auto& var : _trajectory[node_idx].causal_arcs.outgoing) { set new_outgoing_indices; for (const auto& arc_out : var.second) { if (new_indices.find(arc_out) == new_indices.end()) // Arc points to without current sub-trajectory { new_outgoing_indices.insert(_trajectory.size() - 1); // Arc at the End action _trajectory.back().causal_arcs.incoming[var.first] = node_idx; } else new_outgoing_indices.insert(new_indices[arc_out]); } var.second = new_outgoing_indices; } } } void Trajectory::causal_annotation (const ActionModel& action_model, const set& goal_variables, const set& transfer_variables) { set state_variables; if (transfer_variables.empty()) { for (const auto& var : _trajectory[1].state) state_variables.insert(var.first); } else state_variables = transfer_variables; for (const auto& var : state_variables) { int tail = 0; // Start action for (int head = 1; head < (int)_trajectory.size() - 1; ++head) { // Check that all states are properly specified if (_trajectory[head].state.size() != _trajectory[1].state.size() || _trajectory[head].state.find(var) == _trajectory[head].state.end()) throw HierException(__FILE__, __LINE__, "Malformed state in input trajectory."); //set checked_vars = action_model.context_checked_variables(_trajectory[head].action, _trajectory[head].state); set checked_vars = action_model.checked_variables(_trajectory[head].action); if (checked_vars.find(var) != checked_vars.end()) { _trajectory[head].causal_arcs.incoming[var] = tail; _trajectory[tail].causal_arcs.outgoing[var].insert(head); } if (_trajectory[head].state[var] != _trajectory[head + 1].state[var]) // Variable's value changes according to next state tail = head; } if (_trajectory.back().state.size() != _trajectory[1].state.size() || _trajectory.back().state.find(var) == _trajectory.back().state.end()) throw HierException(__FILE__, __LINE__, "Malformed state in input trajectory."); if (goal_variables.empty() || goal_variables.find(var) != goal_variables.end()) { _trajectory.back().causal_arcs.incoming[var] = tail; _trajectory[tail].causal_arcs.outgoing[var].insert(_trajectory.size() - 1); } } } void Trajectory::relevance_annotation (const ActionModel& action_model) { set state_variables; for (const auto& var : _trajectory[1].state) state_variables.insert(var.first); for (const auto& var : state_variables) { int tail = 0; // Start action for (int head = 1; head < (int)_trajectory.size(); ++head) { // Check that all states are properly specified if (_trajectory[head].state.size() != state_variables.size() || _trajectory[head].state.find(var) == _trajectory[head].state.end()) throw HierException(__FILE__, __LINE__, "Malformed state in input trajectory."); if (_trajectory[head].action != End_Action) { set checked_vars = action_model.context_checked_variables(_trajectory[head].action, _trajectory[head].state); if (checked_vars.find(var) == checked_vars.end() && _trajectory[head].state[var] == _trajectory[head + 1].state[var]) continue; } _trajectory[head].causal_arcs.incoming[var] = tail; _trajectory[tail].causal_arcs.outgoing[var].insert(head); tail = head; } } } void Trajectory::cleanup () { // Remove actions that do not affect the outcome for (unsigned a = _trajectory.size() - 2; a > 0 && _trajectory.size() > 2; --a) if (_trajectory[a].causal_arcs.outgoing.empty()) { // Remove incoming arcs for (const auto& in : _trajectory[a].causal_arcs.incoming) { auto out_itr = _trajectory[in.second].causal_arcs.outgoing.find(in.first); out_itr->second.erase(a); if (out_itr->second.empty()) _trajectory[in.second].causal_arcs.outgoing.erase(out_itr); } // Reindex the causal arcs to adjust for the removed action for (unsigned b = a + 1; b < _trajectory.size(); ++b) for (auto& in : _trajectory[b].causal_arcs.incoming) { auto out_itr = _trajectory[in.second].causal_arcs.outgoing.find(in.first); out_itr->second.erase(b); out_itr->second.insert(b - 1); if (in.second > int(a)) --in.second; } // Remove the redundant action _trajectory.erase(_trajectory.begin() + a); } } set Trajectory::incoming_variables (const set& action_indices) const { set variables; for (const auto& ai : action_indices) { set in_vars = extract_map_keys(_trajectory[ai].causal_arcs.incoming); variables.insert(in_vars.begin(), in_vars.end()); } return variables; } set Trajectory::checked_variables () const { set variables; for (unsigned a = 1; a < _trajectory.size(); ++a) { set vars = extract_map_keys(_trajectory[a].causal_arcs.incoming); variables.insert(vars.begin(), vars.end()); } return variables; } set Trajectory::changed_variables () const { set variables; for (unsigned a = 1; a < _trajectory.size(); ++a) { set vars = extract_map_keys(_trajectory[a].causal_arcs.outgoing); variables.insert(vars.begin(), vars.end()); } return variables; } set Trajectory::outgoing_variables (const set& action_indices) const { set variables; for (const auto& ai : action_indices) { set out_vars = extract_map_keys(_trajectory[ai].causal_arcs.outgoing); variables.insert(out_vars.begin(), out_vars.end()); } return variables; } set Trajectory::extract_actions () const { set actions; for (unsigned a = 1; a < _trajectory.size() - 1; ++a) actions.insert(_trajectory[a].action); return actions; } set Trajectory::extract_actions (const set& action_indices) const { set actions; for (const auto& ai : action_indices) actions.insert(_trajectory[ai].action); return actions; } int Trajectory::causal_action_index (const int& action_index, const int& variable) const { auto var_act_itr = _trajectory[action_index].causal_arcs.incoming.find(variable); return var_act_itr != _trajectory[action_index].causal_arcs.incoming.end() ? var_act_itr->second : -1; } unique_ptr Trajectory::goal () const { unique_ptr sub_trj(new Subtrajectory); set pre_vars = extract_map_keys(_trajectory.back().causal_arcs.incoming); for (const auto& var : pre_vars) sub_trj->preconditions[var] = _trajectory.size() - 1; return sub_trj; } unique_ptr Trajectory::extract_subtrajectory (const set& action_indices) const { // Absorb new actions connected via causal arcs labeled with the abstraction // as long as the action does not affect another outside of the current subtrajetory unique_ptr sub_trj(new Subtrajectory(outgoing_variables(action_indices))); map> search_nodes; // action index -> set of variables to be expanded // Add the causal actions to the search graph for (const auto& ai : action_indices) search_nodes[ai] = incoming_variables(ai); bool expanded; do { expanded = false; for (auto& sn : search_nodes) { for (auto v_itr = sn.second.cbegin(); v_itr != sn.second.cend(); ) { auto causal_action_itr = _trajectory[sn.first].causal_arcs.incoming.find(*v_itr); if (causal_action_itr == _trajectory[sn.first].causal_arcs.incoming.end()) throw HierException(__FILE__, __LINE__, "Unknown incoming arc."); if (search_nodes.find(causal_action_itr->second) == search_nodes.end()) { // Ignore the Start action if (causal_action_itr->second == 0) { ++v_itr; continue; } // Check for outgoing arcs that affect nodes beyond the search graph set searched_actions = extract_map_keys(search_nodes); auto out_itr = _trajectory[causal_action_itr->second].causal_arcs.outgoing.cbegin(); for ( ; out_itr != _trajectory[causal_action_itr->second].causal_arcs.outgoing.cend(); ++out_itr) if (!includes(searched_actions.begin(), searched_actions.end(), out_itr->second.begin(), out_itr->second.end())) break; if (out_itr != _trajectory[causal_action_itr->second].causal_arcs.outgoing.end()) { ++v_itr; continue; } set in_vars = incoming_variables(causal_action_itr->second); search_nodes[causal_action_itr->second] = in_vars; expanded = true; } sn.second.erase(v_itr++); if (expanded) break; } if (expanded) break; } } while (expanded); // Determine open preconditions bool search_nodes_spliced = false; for (auto sn_itr = search_nodes.crbegin(); sn_itr != search_nodes.crend(); ++sn_itr) { for (const auto& v : sn_itr->second) { map::const_iterator op_itr = sub_trj->preconditions.find(v); if (op_itr == sub_trj->preconditions.end()) sub_trj->preconditions[v] = sn_itr->first; else if (_trajectory[op_itr->second].causal_arcs.incoming.find(v)->second != _trajectory[sn_itr->first].causal_arcs.incoming.find(v)->second) { // Variable precondition does not lead to the same action within the trajectory search_nodes.erase(search_nodes.begin(), sn_itr.base()); // Removing previously incorporated actions search_nodes_spliced = true; break; } } if (search_nodes_spliced) break; } if (search_nodes_spliced) // Add the remaining incoming arcs to the preconditions for (const auto& in : _trajectory[search_nodes.begin()->first].causal_arcs.incoming) sub_trj->preconditions[in.first] = search_nodes.begin()->first; sub_trj->action_indices = extract_map_keys(search_nodes); // Collect action indices // If seed actions have not survived the precondition check, return empty subtrajectory if (!includes(sub_trj->action_indices.begin(), sub_trj->action_indices.end(), action_indices.begin(), action_indices.end())) return nullptr; return sub_trj; } unique_ptr Trajectory::splitter () const { unique_ptr sub_trj(new Subtrajectory); sub_trj->action_indices.insert(_trajectory.size() - 2); set precondition_vars = incoming_variables(*sub_trj->action_indices.begin()); for (const auto& var : precondition_vars) sub_trj->preconditions[var] = *sub_trj->action_indices.begin(); return sub_trj; } bool Trajectory::always_true (const Expression& exp) const { for (unsigned a = 1; a < _trajectory.size(); ++a) if (!exp.evaluate(_trajectory[a].state)) return false; return true; } map,double> Trajectory::cumulative_reward (const set& action_indices) const { map,double> state_reward; double reward = 0.0; for (auto a_itr = action_indices.crbegin(); a_itr != action_indices.crend(); ++a_itr) { reward += _trajectory[*a_itr].reward; state_reward[_trajectory[*a_itr].state] = reward; } return state_reward; } void Trajectory::print_gv (const string& filename, const MDP& mdp) const { ofstream out((filename + ".gv").c_str()); if (!out.is_open()) throw HierException(__FILE__, __LINE__, "Unable to write graphviz file."); out << "digraph cat {\n"; out << "rankdir=LR\n"; out << "node [shape=plaintext]\n"; out << "edge [arrowhead=normal]\n\n"; // Nodes out << "0 [label=\"Start\",fontname=\"sans\"]\n"; for (unsigned i = 1; i < _trajectory.size() - 1; ++i) out << i << " [label=\"" << mdp.print_action(_trajectory[i].action) << "\",fontname=\"sans\"]\n"; out << _trajectory.size() - 1 << " [label=\"End\",fontname=\"sans\"]\n\n"; // Edges for (unsigned i = 0; i < _trajectory.size(); ++i) for (const auto& out_arc : _trajectory[i].causal_arcs.outgoing) for (const auto& j : out_arc.second) out << i << " -> " << j << " [label=<v" << out_arc.first << ">]\n"; out << "\n# To line up the actions\n"; for (unsigned i = 0; i < _trajectory.size() - 1; ++i) out << i << " -> " << i + 1 << " [weight=100,style=invis]\n"; out << "}\n"; out.close(); if (system(("dot -Tpdf -o " + filename + ".pdf " + filename + ".gv").c_str())) cerr << "(" + to_string(__FILE__) + ":" + to_string(__LINE__) + ")\n"; } hierlearning/hiergen/hierarchy.h0000664000175000017500000000556012213215110017147 0ustar nevilleneville /************************************************* HIERARCHICAL ANALYSIS Neville Mehta **************************************************/ #pragma once #include #include "trajectory.h" #include "catscan.h" #include "../learner/hier/generic/composite_task.h" class Hierarchy { bool _is_root; int _primitive_action; set _abstraction; Expression _admissibility; Expression _termination; vector _subtasks; vector> _subtask_library; Hierarchy(const set& abstraction, const Expression& admissibility, const Expression& termination, const vector& trajectories, const ActionModel& action_model) : _is_root(false), _primitive_action(0), _abstraction(abstraction), _admissibility(admissibility), _termination(termination) { hierarchicalize(trajectories, action_model); } Hierarchy(const set& abstraction, const Expression& admissibility, const Expression& termination, const vector& subtasks) : _is_root(false), _primitive_action(0), _abstraction(abstraction), _admissibility(admissibility), _termination(termination), _subtasks(subtasks) {} Hierarchy (const int& primitive_action, const set& abstraction) : _is_root(false), _primitive_action(primitive_action), _abstraction(abstraction) {} void hierarchicalize(const vector& trajectories, const ActionModel& action_model); void termination_condition(const vector& trajectories); void state_abstraction(const vector& trajectories, const ActionModel& action_model, const set& goal_variables); vector incorporate_subtasks(const CatScan& catscan, const ActionModel& action_model, const unsigned& index = 0, const bool& precondition_ordering = true, const bool& admissibility_wrapper = false); map> extract_action_abstractions(const vector& trajectories, const ActionModel& action_model) const; template CompositeTask::Subtask task_hierarchy_builder(const MDP& mdp, list& Task_list, map& memoizer) const; string abstraction_string(const set& abstraction, const MDP& mdp) const; bool combinable_tasks(vector& subtasks, Task* target_task) const; bool connected_tasks(Task* source_task, Task* target_task) const; void print_cats(const string& directory, const vector& trajectories, const MDP& mdp) const; public: Hierarchy(const string& trajectory_filename, const string& model_directory, const MDP& mdp); Hierarchy(const vector>>& input_trajectories, const string& model_directory, const unsigned& run_index, const MDP& mdp); template void task_hierarchy_builder(const MDP& mdp, list& Task_list) const; ~Hierarchy(); }; #include "hierarchy.inl" hierlearning/hiergen/hierarchy.inl0000664000175000017500000001230512213215120017476 0ustar nevilleneville /************************************************* HIERARCHICAL ANALYSIS Neville Mehta **************************************************/ #pragma once #include template inline void Hierarchy::task_hierarchy_builder (const MDP& mdp, list& Task_list) const { map memoizer; task_hierarchy_builder(mdp, Task_list, memoizer); } template CompositeTask::Subtask Hierarchy::task_hierarchy_builder (const MDP& mdp, list& Task_list, map& memoizer) const { CompositeTask::Subtask task; if (_subtasks.empty()) // Primitive subtask { // Check for matching task for (const auto& tl : Task_list) if (tl->primitive() && tl->abstraction() == _abstraction && dynamic_cast(tl)->primitive_action() == _primitive_action) return tl; // Create new subtask task.link = new PTask(mdp.print_action(_primitive_action), abstraction_string(_abstraction, mdp), _primitive_action, mdp); } else // Composite subtask { vector subtasks; set abstraction(_abstraction); for (unsigned h_t = 0; h_t < _subtasks.size(); ++h_t) { CompositeTask::Subtask subtask; map::const_iterator s_itr = memoizer.find(_subtasks[h_t]); if (s_itr != memoizer.end()) subtask = s_itr->second; else { subtask = _subtasks[h_t]->task_hierarchy_builder(mdp, Task_list, memoizer); memoizer[_subtasks[h_t]] = subtask; } unsigned t; for (t = 0; t < subtasks.size(); ++t) if (subtasks[t] == subtask) break; if (t == subtasks.size()) { subtasks.push_back(subtask); set subtask_abstraction = subtask.link->abstraction(); abstraction.insert(subtask_abstraction.begin(), subtask_abstraction.end()); } } // New termination = OR(old termination, NOT(admissibility)) Expression termination(_admissibility); termination.negate(); termination.disjunct(_termination); // Parameterization of the termination condition: parameter variables are those in the termination condition but not in the abstraction const set termination_variables = termination.variables(); vector parameter_variables; if (!_is_root) // No parameterization for the Root task set_difference(termination_variables.begin(), termination_variables.end(), abstraction.begin(), abstraction.end(), inserter(parameter_variables, parameter_variables.end())); termination.parameterize(parameter_variables); // Determine agent variables in the termination condition vector agent_variables; for (const auto& term_var : termination_variables) if (mdp.state().variable_name(term_var).substr(0,5) == "agent") agent_variables.push_back(term_var); termination.set_agent_variables(agent_variables); // Check for matching tasks for (const auto& tl : Task_list) if (!tl->primitive() && combinable_tasks(subtasks, tl)) { CompositeTask* comp_task = dynamic_cast(tl); map parameter_map; if (termination.parameter_binding_equality(comp_task->termination_expression(), parameter_map)) { // Combine the abstraction set temp_abstraction(tl->abstraction()); abstraction.insert(temp_abstraction.begin(), temp_abstraction.end()); // Combine subtasks for (unsigned i = 0; i < subtasks.size(); ++i) { if (subtasks[i].link == tl) // Avoiding self-referencing continue; bool unique_subtask = true; for (unsigned j = 0; j < comp_task->num_subtasks(); ++j) if (subtasks[i] == comp_task->get_subtask(j)) { unique_subtask = false; break; } if (unique_subtask) comp_task->add_subtask(subtasks[i], mdp); } CompositeTask::Subtask reused_task(tl); if (!parameter_map.empty()) { vector parameter_size; for (unsigned p = 0; p < parameter_variables.size(); ++p) { parameter_size.push_back(mdp.state().variable_size(parameter_variables[parameter_map[p]])); reused_task.parameters += mdp.state().variable_name(parameter_variables[parameter_map[p]]) + ", "; } reused_task.parameters.erase(reused_task.parameters.size() - 2); tl->update_parameters(parameter_size); } return reused_task; } } // Task parameterization string parameter_size_str; for (auto v_itr = parameter_variables.cbegin(); v_itr != parameter_variables.cend(); ++v_itr) { if (v_itr != parameter_variables.begin()) { parameter_size_str += ", "; task.parameters += ", "; } parameter_size_str += to_string(mdp.state().variable_size(*v_itr)); task.parameters += mdp.state().variable_name(*v_itr); } // Create new subtask string task_name; if (_is_root) { task_name = "Root"; task.link = new RTask(task_name, parameter_size_str, abstraction_string(abstraction, mdp), subtasks, nullptr, termination, mdp); } else { task_name = "Task" + to_string(Task_list.size()); task.link = new CTask(task_name, parameter_size_str, abstraction_string(abstraction, mdp), subtasks, nullptr, termination, mdp); } } // Add new task Task_list.push_front(task.link); return task; } hierlearning/hiergen/module.make0000664000175000017500000000017012205065452017151 0ustar nevillenevilleFOLDER = hiergen OBJ = expression.o model.o trajectory.o catscan.o hierarchy.o FILES += $(addprefix $(FOLDER)/, $(OBJ)) hierlearning/makefile0000664000175000017500000000100512304225013015071 0ustar nevilleneville.PHONY: hierlearning clean CC = g++ FLAGS = -Wall -O2 -DNDEBUG -std=c++0x EXE = hierlearning DOMAINS = bitflip taxi wargus LEARNERS = flat/generic flat/q flat/h hier/generic hier/hh hier/maxq FILES := include $(patsubst %, domain/%/module.make, $(DOMAINS)) include $(patsubst %, learner/%/module.make, $(LEARNERS)) include hiergen/module.make FILES := $(FILES) lib/common.o simulator.o main.o %.o: %.cc $(CC) $(FLAGS) -c $< -o $@ hierlearning: $(FILES) $(CC) $(FLAGS) -o $(EXE) $(FILES) clean: rm -f $(FILES) hierlearning/hierlearning.sln0000664000175000017500000000157612303746004016602 0ustar nevillenevilleMicrosoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Express 2012 for Windows Desktop Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "hierlearning", "hierlearning.vcxproj", "{948C4148-9B3E-47D5-A5DC-373FD22F71AF}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 Release|Win32 = Release|Win32 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {948C4148-9B3E-47D5-A5DC-373FD22F71AF}.Debug|Win32.ActiveCfg = Debug|Win32 {948C4148-9B3E-47D5-A5DC-373FD22F71AF}.Debug|Win32.Build.0 = Debug|Win32 {948C4148-9B3E-47D5-A5DC-373FD22F71AF}.Release|Win32.ActiveCfg = Release|Win32 {948C4148-9B3E-47D5-A5DC-373FD22F71AF}.Release|Win32.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection EndGlobal hierlearning/simulator.h0000664000175000017500000000132212303746004015572 0ustar nevilleneville /************************************************* SIMULATOR Neville Mehta **************************************************/ #include #include "domain/mdp.h" #include "learner/learner.h" class Simulator { public: void episode(MDP& mdp, Learner& learner, const unsigned& num_runs, const unsigned& run_index, const unsigned& num_episodes, const bool averaging) const; void step(MDP& mdp, Learner& learner, const unsigned& num_runs, const unsigned& run_index, const unsigned& num_steps, const bool averaging) const; vector>> trajectory_generator(const string& trajectory_filename, MDP& mdp, Learner* const learner, const unsigned& num_trajectories) const; }; hierlearning/process_results0000664000175000017500000000047712303746004016576 0ustar nevilleneville#!/bin/sh if [ $# -lt 3 ] then echo "Usage: $0 " exit 1 fi octave --silent --eval "coalesce("\""results/reward_${1}_${2}"\"",${3});" octave --silent --eval "coalesce("\""results/duration_${1}_${2}"\"",${3});" rm -fr results/reward_${1}_${2}_[0-9]* rm -fr results/duration_${1}_${2}_[0-9]* hierlearning/domain/0000775000175000017500000000000012205065452014655 5ustar nevillenevillehierlearning/domain/bitflip/0000775000175000017500000000000012304226346016307 5ustar nevillenevillehierlearning/domain/bitflip/bitflip.cc0000664000175000017500000001150512205065450020246 0ustar nevilleneville /************************************************* BITFLIP DOMAIN Neville Mehta **************************************************/ #include #include "../../lib/common.h" #include "bitflip.h" vector Bitflip_State::variables () const { vector state_variables; for (int b = 0; b < (int)bits.size(); ++b) state_variables.push_back(b); return state_variables; } // Variable name to integer index int Bitflip_State::variable_index (const string& variable_name) const { if (variable_name.substr(0,3) == "bit") { int index = from_string(variable_name.substr(variable_name.find_first_of("_") + 1)); if (index >= 0 && index < (int)bits.size()) return index; } throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable_name); } string Bitflip_State::variable_name (const int& variable_index) const { if (variable_index >= 0 && variable_index < (int)bits.size()) return "bit_" + to_string(variable_index); throw HierException(__FILE__, __LINE__, "Unknown variable: " + to_string(variable_index)); } // Arity of a variable int Bitflip_State::variable_size (const int& variable_index) const { if (variable_index >= 0 && variable_index < (int)bits.size()) return 2; throw HierException(__FILE__, __LINE__, "Unknown variable: " + to_string(variable_index)); } int Bitflip_State::variable (const int& variable_index) const { if (variable_index >= 0 && variable_index < (int)bits.size()) return bits[variable_index]; throw HierException(__FILE__, __LINE__, "Unknown variable: " + to_string(variable_index)); } int& Bitflip_State::variable (const int& variable_index) { if (variable_index >= 0 && variable_index < (int)bits.size()) return bits[variable_index]; throw HierException(__FILE__, __LINE__, "Unknown variable: " + to_string(variable_index)); } map Bitflip_State::variables_mapper () const { map var_map; for (int b = 0; b < (int)bits.size(); ++b) var_map[b] = variable(b); return var_map; } pair Bitflip_State::parse (string expression) const { int total = 0; // Delete spaces while (expression.find(" ") != string::npos) expression.replace(expression.find(" "), 1, ""); expression.insert(0, "+"); // Explicit sign for positive numbers at the start string::size_type start = expression.find_first_not_of("+-", 0); string::size_type end = expression.find_first_of("+-", start); while (start != string::npos || end != string::npos) { istringstream token(expression.substr(start, end - start)); int value; if (!(token >> value)) // 'token' is not an integer { if (token.str() == "num_bits") value = (int)bits.size(); else return make_pair(false, 0); //throw HierException(__FILE__, __LINE__, "Cannot parse " + expression); } total += (expression[start-1] == '+') ? value : -value; start = expression.find_first_not_of("+-", end); end = expression.find_first_of("+-", start); } return make_pair(true, total); } void Bitflip_State::read (istream& in) { for (unsigned b = 0; b < bits.size() && in >> bits[b]; ++b); } string Bitflip_State::print () const { ostringstream out; for (unsigned b = 0; b < bits.size(); ++b) { if (b > 0) out << " "; if (bits[b] != -1) out << bits[b]; else out << " *"; } return out.str(); } bool Bitflip_State::left_bits_set (const int& index) const { for (int b = 0; b < index; ++b) if (!bits[b]) return false; return true; } // Odd parity when variable index is odd, and even when even bool Bitflip_State::parity (const int& index) const { bool odd = (index % 2 == 0); for (int b = 0; b < index; ++b) odd ^= (bits[b] == 1); return odd; } //***************************************************************************************** void Bitflip::initialize (const bool& target) { for (auto& bit : state().bits) bit = 0; } vector Bitflip::actions () const { vector mdp_actions; for (unsigned a = 0; a < state().bits.size(); ++a) mdp_actions.push_back(a); return mdp_actions; } void Bitflip::process (const vector& action) { _reward = -1.0; //_reward = -pow(2.0, (int)state().bits.size() - 1 - action); //_reward = -pow(2.0, action); _duration = 1.0; if (action[0] < 0 || action[0] >= (int)state().bits.size()) throw HierException(__FILE__, __LINE__, "Unknown action."); if ((action[0] < (int)state().bits.size() - 1 && state().left_bits_set(action[0])) || (action[0] == (int)state().bits.size() - 1 && state().bits[action[0] - 1] && state().parity(action[0]))) state().bits[action[0]] = 1 - state().bits[action[0]]; // Flip the bit else // Reset all bits with indices 0--action for (int b = 0; b <= action[0]; ++b) state().bits[b] = 0; } int Bitflip::action_index (const string& action_name) const { if (action_name.substr(0,5) == "Flip_") return from_string(action_name.substr(5)); throw HierException(__FILE__, __LINE__, "Unknown action name."); } hierlearning/domain/bitflip/module.make0000664000175000017500000000012112205065450020422 0ustar nevillenevilleFOLDER = domain/bitflip OBJ = bitflip.o FILES += $(addprefix $(FOLDER)/, $(OBJ)) hierlearning/domain/bitflip/bitflip.h0000664000175000017500000000444112213030304020076 0ustar nevilleneville /************************************************* BITFLIP DOMAIN Neville Mehta **************************************************/ /*------------------------------------------------------------------------ State variables: bit_0, bit_1, ..., bit_n Actions: Flip_0, Flip_1, ..., Flip_n If i < n then Flip_i toggles bit_i iff all the bits below it are set; otherwise, it resets bits 0--i Flip_n toggles bit_n iff the parity across bits 0--(n-1) equals the evenness of n and bit_(n-1) is set; otherwise, it resets all bits Reward: -1 -------------------------------------------------------------------------*/ #pragma once #include #include "../mdp.h" struct Bitflip_State : public State { valarray bits; Bitflip_State (const int& num_bits = 0) { bits.resize(num_bits, -1); } unique_ptr clone () const { return unique_ptr(new Bitflip_State(*this)); } unique_ptr copy () const { return unique_ptr(new Bitflip_State(bits.size())); } vector variables() const; int variable_index(const string& variable_name) const; string variable_name(const int& variable_index) const; int variable_size(const int& variable_index) const; int variable(const int& variable_index) const; int& variable(const int& variable_index); map variables_mapper() const; unsigned num_agent_variables () const { return bits.size(); } pair parse(string expression) const; void read(istream& in); string print() const; bool left_bits_set(const int& index) const; bool parity(const int& index) const; }; class Bitflip : public MDP { public: Bitflip(const string& name, const unsigned& num_bits) : MDP(name, new Bitflip_State(num_bits)) { initialize(); } const Bitflip_State& state () const { return *static_cast(_state); } // Covariant return type for easy access Bitflip_State& state () { return *static_cast(_state); } // Covariant return type for easy access vector actions() const; void initialize(const bool& target = false); void process(const vector& action); bool terminated () const { return state().left_bits_set((int)state().bits.size()); } int action_index(const string& action_name) const; string print_action (const int& action) const { return "Flip_" + to_string(action); } ~Bitflip () {} }; hierlearning/domain/wargus/0000775000175000017500000000000012304226346016166 5ustar nevillenevillehierlearning/domain/wargus/target_maps/0000775000175000017500000000000012205065450020471 5ustar nevillenevillehierlearning/domain/wargus/target_maps/map_24.pud.gz0000664000175000017500000000051612205065450022706 0ustar nevillenevilleQJ@*X B$i&| ꋆ'8qJEd- ~YŸ!a%)pDD@4Ȉvڷ]Jx|}dTDtr_-Ę_Lge*{!Tlq<`NzUHɡ|RGz<9SFi8K~yJy*xfUL}Ռrճji_B]ΝqXzoZk0 Ӳ]>s7r[MU7{;Z$.x yMqG4k}R 9[,%hierlearning/domain/wargus/target_maps/map_44.pud.gz0000664000175000017500000000053112205065450022705 0ustar nevillenevilleJ@jE #D*b讍nA7RݔlYS m9302WYė"3/C)OAG|~zݖ"y/gwbP'eBOTlp{rH#,3d+:_qi}ɣ$:|wRI#(NL|~% 㡴ެ*c_eը2̧Z.U>/R!*VjTS[Q*T\Y0 g{TcPS-~}HmOz'q\,G|XE,%hierlearning/domain/wargus/target_maps/map_71.pud.gz0000664000175000017500000000050312205065450022704 0ustar nevillenevilleJ02t y/Jjں ވFγT,"v؋')":V="{DD^LYJi}*mv !=>݋ߌC\7q!Hp2zD[U2_ˮ$4p? ߒqC?H鍭24Tv3Ҽ$斳Ҩ+Rқ׶xHo:Ψ+RҼd =cҡ.]pݡ=rx_sM5,%hierlearning/domain/wargus/target_maps/map_39.pud.gz0000664000175000017500000000050112205065450022706 0ustar nevillenevilleJ@Ǥz!*x} !q ^DzRm} t(-MXHo2C˒#.EDܡ{O N_\O|"c4դC{OK'dS6{uFt@$tl|E$VRۮ3 oQn&97(DOoЩQM'Wg{]J&w)̅QM'Wg{]J*n$SX3m6 >&Gԣ[]rӡuʯ>'Wx,%hierlearning/domain/wargus/target_maps/map_43.pud.gz0000664000175000017500000000047312205065450022711 0ustar nevillenevilleJ0oCA& ^ JjҺ ^Dz1 >Ô( OBK[_UjHD̜27Nt`IeLmE~r.ſMgCdsRbǵYg#@ɍjil^HřMutUq?TR[:PT|npn6ًRCmV*x^m3*:Em>q\L2vn65]c‰ѹow]uesoOMmO>,%hierlearning/domain/wargus/target_maps/map_86.pud.gz0000664000175000017500000000051012205065450022710 0ustar nevillenevilleJ@D/?( ^# nVF$7Z>@e} dӊ$Cns!9eC$M[R4mBnLDsOKvV{t tL/woMT|*N/<ٶ#g1N>r_ҐZp LO\BXfad$y(1S71J꒍)YssNZt쒤eOkW1c3U6ʚsE.LiM#9$2tzE;7 oo3|(L&j*g:T%\|ekփ| {./B!)WOKi*+S]Kut=ui=泹3ڨYSYsJ~rcҧ-:QW>h]U0H~,%hierlearning/domain/wargus/target_maps/map_21.pud.gz0000664000175000017500000000050212205065450022676 0ustar nevillenevilleJ@EIR(O?@"I'F$ U? ίLR 2B.0\A]'ND%@wv!t I/oӻB^ xꤶc2EK\fB!B!B!JNi(?_8iIa_Jse $zzRv_J䥫z950[6CetUV6*?0gl>[\[]fhN͇avǤ6.tGZ5 7uW$,%hierlearning/domain/wargus/target_maps/map_76.pud.gz0000664000175000017500000000052012205065450022710 0ustar nevillenevilleON@@hv`< !hq[7u,SM-}7̟dȮf>Votkeq>y~:VP}xxScֿ&i`kwn cp=^ml|E4?_Xiy4?FgZMHal+> P)t^y+mskҋX)s!H%t^y+mskNdY˜>oY7e{{<ɺkI1iH\˜#~wi',%hierlearning/domain/wargus/target_maps/map_99.pud.gz0000664000175000017500000000051612205065450022722 0ustar nevillenevilleJ0Z2uLvG@K)vMF7Z>@V!eA֫?$6rD4s9 W~~ݸYB܍GLDxȘw-_HbԵ7q3!ۓT*s1$6I=6GsOk +Oqp%|?09~hއd299EZ}u#]Қσگxө W1J%<ΓeI:mY}zYYZyPL޿鐮e1Yr~~ T睺uU]lm,%hierlearning/domain/wargus/target_maps/map_38.pud.gz0000664000175000017500000000046712205065450022720 0ustar nevillenevilleJ@ض G vD ݵQ d^)~G06Y9߇A7ÈױOD3OD$caKW_? %c#<-r߾>ǧ{1zt6LwO˹c*&!TmV|\|E$P0O?\՛:=/CQ:^-خ+ڴl%fOKk6m=|~BgZV][smlǼLvh.8iHd}ܘ7/;#4,%hierlearning/domain/wargus/target_maps/map_55.pud.gz0000664000175000017500000000050712205065450022712 0ustar nevillenevilleJ@ZQ EQJ҉F;~-S>}ۋqb2+}CkG3HH/\D9A$qX'=KQ?VM< WoƆ5fiVuaC6ܙi Fw$g4:!k̴AϓjꙞ>Wf\SM.߿z-!sLGEZ||$uuR_8E 1,%hierlearning/domain/wargus/target_maps/map_3.pud.gz0000664000175000017500000000051512205065450022622 0ustar nevillenevilleJ0Nу"(xK/@K)vi&x"R-^@*6t(@J (O~}"D"[< pa6fC9{|Cd<JoPS&d- GD`"/XZp(߽̒sʍㄓ bP4U-[Sb檵]BS]k#?W3r+0sZȞ't֮bm5y5/8?´áګۻӍkv>,%hierlearning/domain/wargus/target_maps/map_19.pud.gz0000664000175000017500000000050212205065450022705 0ustar nevillenevilleJ0c@ d ƥJդu)ꍖз)(w OHekzxC"J=L*ޣKO XpOO\7/N9e2u5R]i9s "!WRyrDC* TC%Lxr E,|zWa+6n۲x\DBwVi+7n۲xX{~}?6WUcշeN/8ɐ9U8!?v@|T3T),%hierlearning/domain/wargus/target_maps/map_54.pud.gz0000664000175000017500000000051512205065450022710 0ustar nevillenevilleJ@Ǥ=(XP|1vC$m$^4T}S4NFTBCw!X*t,"XvHS Mt0I' T:M{rĈLc78ʞ!@8'GD[2WtOq/M}'= @GqB'".\,w #}-2=~d׷g6f(;U\,,~vd@Q>&5䚞ӗk+f Wukbې1uU䚞Wh**fb> S ]j暞R]5^Zw)}kbҩmO#ɒgHcb9z|*ҹ?,%hierlearning/domain/wargus/target_maps/map_48.pud.gz0000664000175000017500000000051412205065450022712 0ustar nevillenevilleJ@vT #Dَ6jA7Rݔluم mIZ w !XGdDU'"rcǻEW6? %#_wo_ӽt+ 6rgv [YL#"&s%I ??؉JOg\&Ahs8臢t ~zchU UQvQ<\FBUdƾҨ+kuU]=ϣo) A\UUx].z!킷mpn'ZuҾˆs,%hierlearning/domain/wargus/target_maps/map_57.pud.gz0000664000175000017500000000052212205065450022711 0ustar nevillenevilleJ`ƏE?D ^B""nmA_"Rۮbvbqx~r^9rx5#"zB;"ꕃݸZ|Ot1{Ȕw͞_Ր Ɠ2ue\mOp'J|}dǂWyzxY^7"zmw7bߦZ/!BsjYD=[*9 ΤK\ OV},ߦ/y໷D:G}m\==)ur~afYH4'iODD]ZxfdYm~W?>݋0u`YTxhT<8=""RDZ/,O?ef&*k$%œ޴*b9+f>W;,VIܜ-L\LWr|nůEwYٓ9#{Y}-xF<_.įEwY\!wfcc2]K>y hơy7onw'p,%hierlearning/domain/wargus/target_maps/map_74.pud.gz0000664000175000017500000000052212205065450022710 0ustar nevillenevilleJ@ѶQ@_)dc轄Cz!7Uh$͑HeF6=~ѥ6"=R: L:Dtr_MD_ G3i%G9CũX*U!OQ[!n& %l<93Nad%Ci1Nެ#ξڳwu9;MԞ7H\7j}gVsBwLi<ё=O}L]}nNw] cS␷9]?Գqg1R۴#%,%hierlearning/domain/wargus/target_maps/map_68.pud.gz0000664000175000017500000000051012205065450022710 0ustar nevillenevilleJ@Rx >!nIڸIloDzm}^( Q^w$gI~="L$AJ[UW[tj#a<S9{~y}3ҦJgR%b1yp~B!RDZv}CUˢX 3R_j+H6@qC?DM_4ea6.]>k:rP,6ƥcqu't5kQ-*6ƥcߧ[1T)/}i@:uH;&vi=:;~S/X,%hierlearning/domain/wargus/target_maps/map_82.pud.gz0000664000175000017500000000052012205065450022705 0ustar nevillenevilleQJ@'IX_؄Iu DKqRv,Lf`ِ2#NDKn$w*TY_J/ӻRDo^p4Pu oPcW;8"#"H+:?dqi͢$z-S+#0NL|~,?| ޽r&,i$8ͪm6#ƎT}-׌93gjd{MYcG*yBg5*YM=&td?8?_C7Y.QW]ڧM=?>=,%hierlearning/domain/wargus/target_maps/map_37.pud.gz0000664000175000017500000000051212205065450022706 0ustar nevillenevilleJ@ſƶ #D*b讍nMT7%[==mn$.΅x!wOp87'!Mmu&C;"Y~ӍGH^?ߟˣtʤUULhr1+uhP's`+:Z+ /7GAC Y(yFP/rcg[yCn{VC).Q0vժX[4g%zz? Z,\3I'5ZuٳR:S^&$%x<j|HC= {,%hierlearning/domain/wargus/target_maps/map_45.pud.gz0000664000175000017500000000051012205065450022703 0ustar nevillenevilleJ@I oE<bI8Il7"Ahy 'p&R. !p8C{bH1*t"5燤xxeח}la<M}{8SNh?T=#VhLg+:ľi@ziR\QnG?PLCŧ7:%˵ߩt|mSKyc*T.`rw*XtX׸^}9Ey)˵ߩttjiT:>&ԣK]vOU6T>}:}'@y,%hierlearning/domain/wargus/target_maps/map_53.pud.gz0000664000175000017500000000051412205065450022706 0ustar nevillenevilleOJ@_B+ n !iIb+ FCROa;bKDLt{|" LԑL8!ä+W2Sw!SL_w)n_=*t$J[VW:3S;/Rc TI<,ܢ&3ti(|c?hs0e=$` ^5e7U~F>)//y휼o56byBPZ4k~mPqXtMd'vhOQHq,%hierlearning/domain/wargus/target_maps/map_29.pud.gz0000664000175000017500000000051712205065450022714 0ustar nevillenevilleJ@ (v!*vD ݝFݠnJz7) ͍ |?gz3(^ HDsODɀ.}lu|Zuy>[><ޱLYKoac@9wzHd֙MrOvDeI4Lz^(_t3{jD#4u1Ahbr3rm:vAh<NJFUijYX_tӃ$L6PUK?ӥN]7=;2,%hierlearning/domain/wargus/target_maps/map_14.pud.gz0000664000175000017500000000051512205065450022704 0ustar nevillenevilleN@EEhv`Bq[7uX[834[ty 'G m~-="D$j^< X=U>/Jݾ=ǧ{1DO.[q*6z-?=$r]H.~E,Z?dQe+V7N2PS^k,|UWn2>Kn!vuΪW7 #<>Ͳ?-y|]pt,&q~ota^ iSoކvhǮ#ڦ6'e,%hierlearning/domain/wargus/target_maps/map_96.pud.gz0000664000175000017500000000051412205065450022715 0ustar nevillenevilleJ@Ƨ=X?uAfj>~l_cf_d:ߡ]wQ6٧:˲x bU]e}l&K,%hierlearning/domain/wargus/target_maps/map_6.pud.gz0000664000175000017500000000053512205065450022627 0ustar nevilleneville]J@o~`?,A\"I'HP_4T $өdf*&M bMVp\!sɘ=6S4+dJrI=Oi3j^5٫WhϽgF"=4ۤG>QMӖ>KW~ۊ,%hierlearning/domain/wargus/target_maps/map_15.pud.gz0000664000175000017500000000050312205065450022702 0ustar nevillenevilleJ@E_ >!D6N[HP7Z~@֕`;TD&%D { 0!/SGD#9)p.f'>NO/ӛrzp}cb8'ec:dpBK)B-w|E,,%5KY||(v(CI>NjXgd-;AŠf =f5sbjκx+V=O5}Us֝z(?2d|:3uڦu糒DteZ.֎,%hierlearning/domain/wargus/target_maps/map_17.pud.gz0000664000175000017500000000051212205065450022704 0ustar nevilleneville_J0U ĺ)ִuWE/ZHI2@Hi[^M"}"TmMɸ ErPOb/Ɠ6+ EbM됩Y,#5"UN|S~AP$v";Nq4~T9Uwo+5x'5+]߲U\*!)'uy'u+]߲Ğ't'}\.>r[5\;K(-XH8vI8Il7"Ah n>O:,H;;Cn$合\Dǀb~~HB&#W*}y̿{{Ol&ӱZ#9Azd"gD[u2_ Wt*@Z53/$ᅲ&Ah t1}zc),Vʳ۵mLr1fn9)-Vʳ۵mOjK;kVj7׾d =ģ]G}rh6hHҾߗe,%hierlearning/domain/wargus/target_maps/map_51.pud.gz0000664000175000017500000000051112205065450022701 0ustar nevillenevilleQJ@Zhx1vC$mMl_Dr)EA6YL`fXy4&EdD>L]euSZ?ʩ1_ǛvM".s$}JULJD=&en|cδ~ L*o6T|$$Oʊe&,^}5ۗC_kVysIv9A_kVyn?֭OߺxY_-W?I9K> :qhgGkHG81]{}R,%hierlearning/domain/wargus/target_maps/map_28.pud.gz0000664000175000017500000000051212205065450022706 0ustar nevillenevilleJ@ "B$i&mŇh:݃Rragf0,$MhS) ?=p!  ^}q_\<=?1ۿ&j]in(hU?;&"lRJ7Y$?8YK_ eFϏPӛBQ2sUmSr6]Uvo9/YfڰmVئkRA:el\&N6uݼk*&Q~cq\r9TXnq<js>@wM*=֫fjV7ڛ0 ]|1_\||un]}13&>yt\ }ڵQP=Mo.,%hierlearning/domain/wargus/target_maps/map_5.pud.gz0000664000175000017500000000047212205065450022626 0ustar nevillenevilleJ0ϴ7fJ|NJ35i܈uèw>i1"E`G8is&H6h}i7DoLLOLwZJ8`Mn8OQ`R"Vv:yT˵_(ii}#0i&>i#%"a?\-`[w;\.Ns ao9.V)dNR{!l/FrI`W.t5YgGSWޟCڤs^_u[_Oy,%hierlearning/domain/wargus/target_maps/map_27.pud.gz0000664000175000017500000000051612205065450022711 0ustar nevillenevilleON@P]Ti#` &niԍ6UЛC'y,%hierlearning/domain/wargus/target_maps/map_61.pud.gz0000664000175000017500000000051612205065450022707 0ustar nevillenevilleJ@ m # nV"ԋnQZD0?̐̄. IyGD4 ͉œ8*2ʳׇblf]ׇztt61]_'2sdjCJhNDZxEjw/aIIcf):ʄO褩>8oΆ/o1C-:uT׀:hۭ #W,%hierlearning/domain/wargus/target_maps/map_66.pud.gz0000664000175000017500000000052212205065450022711 0ustar nevillenevilleN@1!Zk4`|#B0&^!EG|&B!롷%;;3_vl@z{"8LDNLd_6xR\?"Kg2{?¿/b?Ɠ,Wڱ35 "x9?!2dt" \O+R^J-8u|Nt >3V\Q=o4XvWvl%ב(ɂeTT;M2֫}mEdY, Xw|scOo=Q˦8C2r^=꫸㝺TP;,%hierlearning/domain/wargus/target_maps/map_8.pud.gz0000664000175000017500000000050412205065450022625 0ustar nevillenevilleJ@1i/J%E>bIڸIl/"Ahz)@vi&a 4eDD4͑*h{:EbTNi_7Jípl>5ċ&tV\-uMjnU~4nO(7rI.CE(қĹq?9_|:qn{VR)K\︟ۜr>S=M R+iNs\G:A5Ӭ>c2댯 )#5خ`8,%hierlearning/domain/wargus/target_maps/map_67.pud.gz0000664000175000017500000000047312205065450022717 0ustar nevillenevilleJ@?P# NL'F$-U.>f 1wC2$M<}$b.|-#'CΟ_7írd;m6IPu}V&Sh`Z]E$$0T/29/_T:U7nfV=֫n} uGɹœrªz\խO5wH9O褽z/ץ.e0=1р6˿]',%hierlearning/domain/wargus/target_maps/map_69.pud.gz0000664000175000017500000000052012205065450022712 0ustar nevillenevilleJ0O[1t# ֥JŴs)ꍖg%} 46Ӏ>I'!BJ]GDc_DYu.\#}Z2z|,~˧ʿi[$sЏÁ3ÀסGDcK G=r0yl'd6K{NŀPEiM 尽@r<`%vlI\iL|Eȳ?_B3?E<_gyG?ē@ ;LۗUk}lɺMv SFY* P\kj/YNvJݺY>"c&w"{\v,ͪMeYUoe2uAO i@T4"K!/{T;/D2,%hierlearning/domain/wargus/target_maps/map_97.pud.gz0000664000175000017500000000050012205065450022711 0ustar nevillenevilleOJ0,9xNJ4ӚnD2z)P+ZR>): gD AJ4P1ϬN/o뻂E͍E\fo`Y F< NvmH񩏯|~(̶~Wt|D.^ V7ERYvW:xwR2fO9*U9Olwo~Ȟ't CeSoj7ۺ~zwBdԜ']'=5P#[ǴkƬNhf'ۃ,%hierlearning/domain/wargus/target_maps/map_12.pud.gz0000664000175000017500000000047612205065450022710 0ustar nevillenevilleJ0m1P6 >X"VMF7:>^')YeV]tw.J.N(U&D4igog-E i{̭L:-Yrr_NX ff4|P5w"DEĈNb$/eԬtWY~H#͂ v'ܫqahЌ\ȩ{5 6l32: 1[_jw½˓&5mjQO{nҖڥkw~~k^,%hierlearning/domain/wargus/target_maps/map_7.pud.gz0000664000175000017500000000050612205065450022626 0ustar nevillenevilleAN@_!hv`<;`lMCԍl+x,4.X_ x P\g>Ms #OB>@:GGz}~U*}{Oj_&ӱYi]%4̱ zȖNU=_#'m3O΍(7I.y(-fٛ5fvkDiAyl7e]`vkDiAyl7&zBg.E7]>'Q}P5M9c&.tuvȱRCkH,%hierlearning/domain/wargus/target_maps/map_10.pud.gz0000664000175000017500000000051312205065450022676 0ustar nevillenevilleAN@_4&x#B0&n!FIpo|<'1/X;zGZ3M%l/}'lb)QP'q}+:+"V"׷ی+)7.9 OofVc֜ [j=]wctn9[Zsb^o@..EQeUn[Jo@1>]s!i|~4TuH&tGlYp,%hierlearning/domain/wargus/target_maps/map_84.pud.gz0000664000175000017500000000050612205065450022713 0ustar nevillenevilleJ0&#ދu)l5moDzeE)P2DrQ '?? h, hcŒh:yUIFDLW/o˻Jķ^Ll>mзtC&J JYD[2r_t~i8L?7KR_d/CX(7T;jGo휷t}/攃l5ξ8mn%&v_u [+W_]g%&v_iR6Ј.) ۴gjߏy 2u,%hierlearning/domain/wargus/target_maps/map_77.pud.gz0000664000175000017500000000051512205065450022715 0ustar nevillenevilleAJ@gZ n !$܈uMNnŝVb9̺}5ooZ&ל|f؀.cv1WȬ}uD{tg6?m{,%hierlearning/domain/wargus/target_maps/map_26.pud.gz0000664000175000017500000000050212205065450022703 0ustar nevillenevilleJP:͢) G "Us6RmJz߶hF]9Ϗ9mQ31t3˧t fGĽd$׏Cb2_t񄻲~,=?!]2_}_hHiN3=~:}JKa[ (]EL>1e=JC_㏍s:bLo9CU]z eUo綽> C_mU e t?moSBH"Wv[wL8,%hierlearning/domain/wargus/target_maps/map_47.pud.gz0000664000175000017500000000051512205065450022712 0ustar nevillenevilleJ0DL#ދu)l]ں ވDe>ź,2hZZ8߹Pz~Mv" ?!)N{tHătHo/󻂅 t$Wor ?sGU=Ty䞋F\iIØ3=r7)/o&>?JD6Lޤa*:Ic3s2҉`LrSiWOM}5l9'P'tެ4uՓf.$nS_M;[e(;~MK^AэzѾKqG,%hierlearning/domain/wargus/target_maps/map_88.pud.gz0000664000175000017500000000052212205065450022715 0ustar nevillenevilleJ@jDmt;=@d;"ڨDHuVm ʍY*8Q51]u*"7&]/!]{JX<~HK&;,V^󃘰|{O K;dGL1рI+9@Z?BvBfAhs8~(J7h #/ -Rw](Ч\ʖ4r=ViaEJN ) 7c$n/N'mͺ>mZ̟:P2 u,%hierlearning/domain/wargus/target_maps/map_4.pud.gz0000664000175000017500000000047212205065450022625 0ustar nevillenevilleAO0zXY=׆ x1]R [؉fhKevhc}}.?'LWϏJD/wV d:u5Ùg2N*V)hLiy,?_xyHa?əMY<TZɨl+l{PߕB[]oFMf[aۻ$\."sЩ>̶¶wI]8.l88z|~I{&S,%hierlearning/domain/wargus/target_maps/map_2.pud.gz0000664000175000017500000000050112205065450022614 0ustar nevillenevilleJ@JOU# "6n[HP/*UUZJr(ڡtgYM~D4Vv_^{tZ)ma6|eˇe)rzs{%m4oZ VMб];B`CGkLE%+YrdFqӧ4=d|zӖ~_߷:֝)]yǯTί[a't؎Ͼٖw~MXw1=>&]1׀IYV]9F[~V,%hierlearning/domain/wargus/target_maps/map_34.pud.gz0000664000175000017500000000052512205065450022707 0ustar nevillenevilleQJ@E_% c'HҎVA?Z.qmv2M4Q_rx%$^|w~3OQ&97aP "vЎrEH îM|Su6>OIU*RaתMZ+%|<̏WEjSN_i>i7H6٦6$[&ڠ[UT),%hierlearning/domain/wargus/target_maps/map_56.pud.gz0000664000175000017500000000050512205065450022711 0ustar nevillenevilleJ@'"Tҋ71vC$Il/"Ah}3ǧh,DJx f~vؐ7*hHE4N?<p!0 ^}y_\>=?1&5 u0wMBƩz2Ύ@"r+:d4fQ"Y~zmfɅM&*z~:Ǜ,%hierlearning/domain/wargus/target_maps/map_49.pud.gz0000664000175000017500000000052012205065450022710 0ustar nevillenevilleJ@Ə{nlG]ݠnJz)ڎŶw}1 hz{{D4u͉ܘh}3i/)m~+!?>ݫLǦ,wfD=="d;KJs#MPs8lxYxƝR!t"g!/.ڤ\FJջ\T ]Y >hKA96iϯ:>2W%o6^5'&lǼLf.nSlQyt׷S/6,%hierlearning/domain/wargus/target_maps/map_52.pud.gz0000664000175000017500000000050312205065450022703 0ustar nevillenevilleJ0v(xG@K)fI&x#R-S"x>3 RY)RF(wqrr.BaKnbDD3O2D}!q_~>ccxz~`c^&r]Ko H'wvL#X]GU <~EEQ4ya՜D2Ldr}=۞"dyꊺ {=EɊ2}BHݷ=۞}?ɉ&4K]O?{coшnLu,%hierlearning/domain/wargus/target_maps/map_22.pud.gz0000664000175000017500000000052012205065450022677 0ustar nevillenevilleJ0*CTm@K)fi&x37*S)i`uwm}$$%$>M|}*y]Z8d)}~WpPФOFR$Ա2Rk16 6C-.&' c)l4Ьqs(l{/dwSV]UZOv[alU?n*ָJ:~BKeOn:ʟ5z>G%u`K6zԣ[S72~1Z,%hierlearning/domain/wargus/target_maps/map_93.pud.gz0000664000175000017500000000051012205065450022706 0ustar nevillenevilleJ@o+(ZA# "Ig[HP7>@ֽ{4.;3CH2!Iƞ:mΎ?W$}=R.C>}|0J]w_y$d5CV&S숨CX$ /:DR廻L::WVQ0I,xz**%5O@g׋+X˕:5O@g׋Ε+)?0 ?gߢr'Eaճe0=1٦ `gwx~*юu>n/5a,%hierlearning/domain/wargus/target_maps/map_41.pud.gz0000664000175000017500000000051212205065450022701 0ustar nevillenevilleANPGE5}G0 7ubRb&f›<Uh Q~wHq]ׇbtFlb2J{:vqxav9r/sOv_& =OVD2\M¨ z(MUJʕJRfUŒvJW(VnjKOLVzYK;+5]俺?0;1P.L35n̛c'n,%hierlearning/domain/wargus/target_maps/map_70.pud.gz0000664000175000017500000000051612205065450022707 0ustar nevillenevilleJ@OkՂ# "I'F$^6>@ԥOaT$=1gH~!2 D4H2#Yܾ"7HH^7_7X/;UB-gM8KU"Ji+:?䱲~Qmɩf&q8KJY*X[3֚js7?Owvǐڰs\tN0:ict]+In 99rflSL&f g9w!]'5O.{,%hierlearning/domain/wargus/target_maps/map_0.pud.gz0000664000175000017500000000052112205065450022614 0ustar nevillenevilleJ@?ǖhYN lG]ݠNJX"@yf_>AM#wf< ;+WrH񘫙^wn_=7D; *tL' 5*S8GDJIm-:?ľߴj ?\~`s( yjʨ|ms.jͼ`݊/CSRi5\ۜZsZU{MnI:*VE}e$i&w3 viޭt'Ј.iDL=׸GF[to@*,%hierlearning/domain/wargus/target_maps/map_65.pud.gz0000664000175000017500000000051012205065450022705 0ustar nevillenevilleAJ@_VJ(#c'HIb+ Ɔ鼊h|RtoHya I"Ǝ:uݺJXT6=f{xf/ObFZԱWq3tBhs=aS8D-*dWt~}i@ =3Qn&9Ǐ5PLB726g]΍sr՞uz5 з8W(WYw\.M<~ܦiڳNﰹC?9cҦ.]^9$K];ӵi_o57,%hierlearning/domain/wargus/target_maps/map_72.pud.gz0000664000175000017500000000051512205065450022710 0ustar nevillenevilleJ@O[Et;=@d;"ڨD,M>ouv؈:? jvD4̉hz_nӕoׇtxd/\ӽtlN}wx3:tCV3hN=poPZ?p ܿFF䛄IPbūD}ENR'E¬4KzzIԶlIꄺRR&y1~`UEg[UNR'Eɞn&yavL[]pӐ>-:u7<~>Pj,%hierlearning/domain/wargus/target_maps/map_18.pud.gz0000664000175000017500000000050312205065450022705 0ustar nevillenevilleAN@v&#`!hlMCԍ6U{zOa^6QF-ü @y}"GD"/wztkɸC={|_1us`XG2ŖJGؤJ |E`MJ H2)xek(\rQ2<\WU{ʽ7=6e.?YU{ʽ7︃=6a'uΖ~.ǿiOp?\ ŸVô<7>]p%?r in٦/m,%hierlearning/domain/wargus/target_maps/map_32.pud.gz0000664000175000017500000000052012205065450022700 0ustar nevillenevilleJ@'I롨(# l]w[HP/Z>@֧nTL%;̒%W*#QE*&͹E4; ő=&[й]oOϮXCt/,%hierlearning/domain/wargus/target_maps/map_92.pud.gz0000664000175000017500000000051112205065450022706 0ustar nevillenevilleAJ0팢q@mN :)ԤuFp#RԍQxMO2"eeS^!opLD@4ȈV#3Rc|z~a6fIg2NCB`"Rr/:-:N)N.+7N.4_¬ *U֯z1U*堾f\ݙkpKJOꋾV~=I4,%hierlearning/domain/wargus/target_maps/map_80.pud.gz0000664000175000017500000000053212205065450022706 0ustar nevillenevilleQKP邊ݞ]t1tk dlb,j/Pv>sܧҫ%TDq.=,>Gt:͔rMT ]]i^2 7T}vQnƯ|OV⇮2w7Lr <Hs6)MK4]sa򚫾;N."*WY-dJ֨cf=Y9+&)U6ۯjg\AzȯItNպ#M~ck>u2,%hierlearning/domain/wargus/target_maps/map_35.pud.gz0000664000175000017500000000052512205065450022710 0ustar nevillenevilleJ@Fo  1%iIb+ FCۺo(MW;r˝hhShܥk_ 'sܽ=泧1HU6e|N:ᅱq=b3/uώ,2Y=KzZ $ t:-Dxr_D_]]ީ-hxe9EP]_&7 c)L4>V^Hſ<% <9BZVٮvdK/!.WwB]Vxɧ'tZwMs*_*}y7ʎxL6E7I.OY2׮h ,%hierlearning/domain/wargus/target_maps/map_62.pud.gz0000664000175000017500000000051112205065450022703 0ustar nevillenevilleJ0O[0Q' RJbں ވFH>O.M vI?7!d"ȗ,Q] Oi,>޿O ?_3@VoxM]QvT=U,%hierlearning/domain/wargus/target_maps/map_87.pud.gz0000664000175000017500000000051312205065450022714 0ustar nevillenevilleAJ@_HU*#c'HҎVp#ԍȵܙ-㵩.XZ!͇;K:X{mlg 쐮MRwcѽޣq#,%hierlearning/domain/wargus/target_maps/map_16.pud.gz0000664000175000017500000000052312205065450022705 0ustar nevilleneville]J@oC(:K NVE$/6T]@V\u2CEB|p0&?uh:mCW~@ҍGBT<]e»}yOb_LcUvڲq3Cz9="I=%6P{fn88W,8>ǏPBި,OEl_;yc6[uVPu)XN*{XVUoznzf?.xJ|䮲n+VUozt'ǫg }'˼CmZ˺Othh',%hierlearning/domain/wargus/target_maps/map_30.pud.gz0000664000175000017500000000050712205065450022703 0ustar nevilleneville]J@o~TPB$iLb+"2ܡ Zi@j|pK2eG,]>q_r"~|,~glKƓ(&m8›:Ø(e<D`,/:Ȑ3 "δt$S(7&9/L?t3&PeViJ֩uUۜ<ӻ-kUgRv9ujk6'@'tuK_Bnՙ]}ٴVms(LBrLUz[E\}ڣ+}5s,%hierlearning/domain/wargus/target_maps/map_42.pud.gz0000664000175000017500000000051212205065450022702 0ustar nevillenevilleJ@ZDK,EAD#b讍nMT7%[=~q;raK"wHD3[Ў.]L~@}q)1[<=?1ǿ&Գq=7j!:}zDdI=Zobߩ{/LrS?l? ~gs|Tu,%hierlearning/domain/wargus/target_maps/map_40.pud.gz0000664000175000017500000000046612205065450022710 0ustar nevillenevilleJ@ǤC(xG@V6&`h};ŧn֠4,seOf.aBRɘ:<.FeD׉I$z&SuYxb?SixX9L<2DvR%_EKaiut Gtv|3bYdy(/9 VV~v=ʙ햃 6V~v=JRO\6ߵ[orwɥ}ۧ6M1\=/,%hierlearning/domain/wargus/source_maps/0000775000175000017500000000000012205065450020503 5ustar nevillenevillehierlearning/domain/wargus/source_maps/map_24.pud.gz0000664000175000017500000000041612205065450022717 0ustar nevillenevilleAJ0gt8 %ԤuFp#Rԍʨ& )"Y to*sHD O٩htDs&'M|o{lWOn*]LY%hierlearning/domain/wargus/source_maps/map_39.pud.gz0000664000175000017500000000042012205065450022720 0ustar nevillenevilleJ@1RQ.a%ۦEJP/*UZsP*M2 IySM?g eCs6aqż^<>0OG~tjH;q=t.NUׇ) V?enK~wS7r8No -ʋ~aCC\?o29OAړ*FCSC(l.\#a#it}R:L|iDqr,nńǿSR?ߤ\dqB+x=jUCjWtoY(J%#P)Sq/CZXu'.m!\8Bħ\7zz%u'tҍVM7FQ=5՘.C >+^?jY%hierlearning/domain/wargus/source_maps/map_86.pud.gz0000664000175000017500000000042712205065450022731 0ustar nevillenevilleJ0k(>wbifj:#)FQ`G)fq|PBJKDChcvVrz,>?Nn[o^e0'el#y^q_ҘZ,j`3|EV9i0_"]Y>gٽ%1t.S8zi/4-7:$glDFtt,G1IFG|~|]KN^r#_6fxbԆ|S\ D-mEbe?WtNO%0K/NNw <|fؿR9O sX吪yk9R5fzH):? ;}l Qn;/%hierlearning/domain/wargus/source_maps/map_89.pud.gz0000664000175000017500000000042612205065450022733 0ustar nevillenevilleJ0QPA]JٺuF7:>֧pfP&"(|ҴJsQRjV"HwVD8 OIi?0rY>> Կ&CҞئfn؟ԪDmuԑWtoިO8|}"'MYr?dRɽɏ-s\{y\O A:CSX7/$֡cڧy8.2%hierlearning/domain/wargus/source_maps/map_50.pud.gz0000664000175000017500000000041512205065450022715 0ustar nevillenevilleAJ@gbt ucg#6$m7"Aݨ]Ov:HW709b#rrPY دl>N_3|}c0.e8 )e!#]eji[NMȥ䰜e7v@gγ_L16wo 5z0SAꑯJyBR,+$ 3J-:?Wj%hierlearning/domain/wargus/source_maps/map_21.pud.gz0000664000175000017500000000042412205065450022713 0ustar nevillenevilleJ0g(#˷wqJIFQ?+B)uBʥ@(mLSv"u;'Ds&H0fcqrq.x,į3^ͳ񦌇Fh`]Sx+:`uGRs~%>7eJO٪rܱ t^%i=U_•cO9dV}:GezX4[6ٕ=s)H,Cmӭl%hierlearning/domain/wargus/source_maps/map_76.pud.gz0000664000175000017500000000043012205065450022722 0ustar nevillenevilleJ0!P=~ϰ'1.Dڭ&ES`{o9(\,;C'm}U]"(7RUD[Fq"I?du~r~rx(/ȭVS$4-y[IkurHEɝR|E~H|^hEƶ)[zZ A:,7ۘ=9rMѽٙ<'t cۋm䘤ԣswdLQF %hierlearning/domain/wargus/source_maps/map_99.pud.gz0000664000175000017500000000042512205065450022733 0ustar nevillenevilleAJ@gҺB]3HƙVp#ԍ2S4>T$:߿$ ETץ'rc.TI4jC2~H 5/˷]Cz|S=0fZ$m9Ij]cmu^rav.7yLO2~["Hg%q>_aCKz}\3YO Anm\:~rxFξ=/%hierlearning/domain/wargus/source_maps/map_38.pud.gz0000664000175000017500000000042212205065450022721 0ustar nevillenevilleJ0 у @K(fI&x)EeF>oDdO=$-hsoD4VဝD[.!meUM3,2C{^>5 mweܕj]=z;WtoO,4}*au#[.NY_r?䪉cy{-]OxBIg"i9lBlU%h2lj#5/oׇft= kt^( m,/˸/uNչ)FZ$>/4{_|qzS)?CY[85^`mVe\L#[FneyOm>fqt9uTrLh.d\-JPJ7 =#%hierlearning/domain/wargus/source_maps/map_31.pud.gz0000664000175000017500000000042112205065450022711 0ustar nevillenevilleAJ@gZP;F2m nDQz/SX_TDK/f?!@xQ>?d*Y|x.i,;<=L.o m46eƫI:^uOR먎Յ׽o~7_P=/<+F\lU<ıLo _yl3ǜN9Y䵥^M:FE>fzhが6T-ڡJ}KY?<1%hierlearning/domain/wargus/source_maps/map_3.pud.gz0000664000175000017500000000043212205065450022632 0ustar nevillenevilleJ0Zt( >[B4[LZ7HQ/*S|տ9L ɵ|ʟK(lՍUGD4ݹ0%WFʉT~"x皷}#f &<6OM$LHm'\-|tJ4 TN2Ց"lEt3k[x2>vWF7PWF "B9U ]t[vꮹ)\=t>?)n?%hierlearning/domain/wargus/source_maps/map_19.pud.gz0000664000175000017500000000042112205065450022717 0ustar nevillenevilleJ@1Q AQo7X/eS?Ph wh*m[t.)FWy~|4~kb_&ӱ_ީ^β8|jUɣ}j+:`TCR|+'>inbʔ9i\fVuĥN:֠pfozR'{A^}Bݵhmdυ_cS6АN8UgtP ]%hierlearning/domain/wargus/source_maps/map_98.pud.gz0000664000175000017500000000043512205065450022733 0ustar nevillenevilleAJ@gbTDP;F2mIl7"R7p84bd$S!!2uDDaوh VY$?\ X^s|E~BQj'FL)[\=q,BN&kp- ]2Olrw>钏EZ+ m}(ǤGɘQvJOkă%hierlearning/domain/wargus/source_maps/map_59.pud.gz0000664000175000017500000000041712205065450022730 0ustar nevillenevilleJ0y1Qin.CEe,OLs{H(m;D4nFXl1V9NSʏ8\9_Gad:v͝6,+6 Y<'Uo~x-7_3ݗ/A&8dжyȔ3aV1OcP̩<'4kԾ"ltc:uio[%hierlearning/domain/wargus/source_maps/map_95.pud.gz0000664000175000017500000000042612205065450022730 0ustar nevillenevilleJ0!@A/3b\B4[MZ7{3ǧpBɭ委@ueD4Qv>]ϙ? meۗ]C;|Կ'#zR?_gf-%Q`]Sx+:`uKRs~%>7eJO٪xrԱ A:3K6x¥cte[l*xS+^j?Xɞ w)L468h%hierlearning/domain/wargus/source_maps/map_20.pud.gz0000664000175000017500000000042712205065450022715 0ustar nevillenevilleJ0va z|1[B4[MZ.2z1>3a2ɡ GI%жꈈ&^Qos~qHwrTn~fחբ^==?׷ƓmMw)+IhHmJ`WYO2|!qY.9L?ҿuYrj?etoq9r^ѽ^Ǫ,'t sۋmo.Ϩ9O:tǮORz4i%hierlearning/domain/wargus/source_maps/map_74.pud.gz0000664000175000017500000000041612205065450022724 0ustar nevillenevilleAJ0gq@A @P"LMZg7"Eݨz <3 eDdv~hB ݄ж(퐈:% Vr#Yh^ WϷGR6J2LqeܓԦGDk3aR|P>Jr(m)D4$>t# ' iGfy{X<4߿?5Gd:6v.iYBiإUv~BCUF REeߴHl^ӯtSlqizU2Ut52̃{l/Z3S.,{˨l/F/4^BmyY]"(NDGs"H0duvSsvrWny(o/VZOIlhxM}IKu|@#XVe^x#Xi, _7*=e!WN-SG=i'1SɼuCvM:i2nzN-MEg2n]Ku%hierlearning/domain/wargus/source_maps/map_37.pud.gz0000664000175000017500000000041012205065450022715 0ustar nevillenevilleAJ@$u .3xq !iLb+ FKx#)Nfa-|ϻ! !0!I}WC#Ū ؛O2'I8~ m|_̵oVv$p4pݙz*8+ta% Iѵ8v@DDDDDDDDDDD郔oщ|7?Z'/_^ ru [bmh7V$rbq?%clټY@]\1CJG}2A%hierlearning/domain/wargus/source_maps/map_45.pud.gz0000664000175000017500000000043212205065450022720 0ustar nevillenevilleJ0ZuLgĸin^T>W>Oʆ=$?Rh[^zHDSNѠڡ+DHi?sC=|#5]{tvr eT)Oz̩ox75r%;ݔ]1s?CYwozኈJ rm\/)YxYۢY4?M!ԕ)6.xӀ?iH7.=d%hierlearning/domain/wargus/source_maps/map_53.pud.gz0000664000175000017500000000043012205065450022715 0ustar nevillenevilleOJ0g0AA @P"LMZg7"E0<x g,lM(%h\fFrjK΍L62s鶛p__xZ85BCۊWeܓQGD+oy72_-+31dB m˻Bh,QuU??&܀֯Y^<>yF]]i?ݤ$!_O|*YS=MUfQkNMeJvH.)'-y2o 튻1)ǖy=iWb^M]<$.]V("%hierlearning/domain/wargus/source_maps/map_9.pud.gz0000664000175000017500000000041712205065450022643 0ustar nevillenevilleJ0[E %l1i/"EL}w0Ka|MKh{ f*Sh{QЅ =&,=?No:_-2}a46e&IfhxCRu|@,k|VZ?䱪5{W> +zy!LA1]RBkP0UO>g.BPI!kL[K +9w%hierlearning/domain/wargus/source_maps/map_11.pud.gz0000664000175000017500000000043312205065450022712 0ustar nevillenevilleAJ@Έ,E]<8v0iǙVp#ԍJx@)T$m{ńQ ŵ{D4DXsHWڋ_? H^wnӽd:w봗Y(c\ yzD#XWf^W0M$dWV}nb4.-fV6¹bwupihr}pbwu'yBg-EiZN|i.Pڧ! }%hierlearning/domain/wargus/source_maps/map_6.pud.gz0000664000175000017500000000043012205065450022633 0ustar nevillenevilleJ0Z8 >P"M"RԋT6>9(s"9u ][^JJ[]:KD鏄hxӅ"{t ]3C66ei'bZUG.x#iT"~S476eʜ4zVS+uĥ9^8BU޹Bk"'tmvަu}eVLutFue2)%hierlearning/domain/wargus/source_maps/map_25.pud.gz0000664000175000017500000000042712205065450022722 0ustar nevillenevilleAJ0࿭D] @M(nDQSL' ,)-ʣ&7> tgʊh ]k'R~DJ\̵o/VwA\1MN&< eӨZ%©QkިMa_^Ԕ1=i![ϭ[! ʉmBSAONlU.yB2,eofbS&#ڥ1G?ڡ[%hierlearning/domain/wargus/source_maps/map_15.pud.gz0000664000175000017500000000043112205065450022714 0ustar nevillenevilleOJ0g ]3x1NB435i܈u2bkʈH,|PoBT\#$;ta'r~k_7 g/w'Rm6J[:" :)㾤֕:> XWgNx:|%Y9L?SU7ˋ:CZX["HAM(7!ж>͔?ajbu ϙ? me滧UMԿg#Nx5bC[2JZݨz/[tM,8}KF6)[W<fXvoD{!'Ls%hierlearning/domain/wargus/source_maps/map_28.pud.gz0000664000175000017500000000043012205065450022717 0ustar nevillenevilleAJ@ΈuEPt9gHgnTz)(IgP>B7!жꀈ;&DVe ͕ DR3׼>Qܾ=4ǧ{1b꾯4L)/ڈ=>s р`[]82GRpv 2fk[yTK#;zlDq+2B\w Bʩ"i#^fFaC[rK.2z;ƯrZDyj{T6]eO,+f=Ssm!3r^멹r$yBNj>?Ssm!ʑ-rMzEr|)O$%hierlearning/domain/wargus/source_maps/map_36.pud.gz0000664000175000017500000000043212205065450022720 0ustar nevillenevilleAJ@gR]t!.xq a$Ib+ FK^SX_T̪?a2!!n GDCO٩;8еҦ|>o8{{l&OܓWNFIhh#㡤֕:;""XVg^:#)\s~R>7yLO"~U#Dz{gojr)L{ X-x =_^lA:RJ2H?2K%hierlearning/domain/wargus/source_maps/map_58.pud.gz0000664000175000017500000000042212205065450022723 0ustar nevillenevilleJ0Z(( {1.D-&ːyQWiDdo}J{(UM`ޣ+8 I*?0s|Ww _ƓkM7)'I\!0*Yc*w-:7eCBrsպ.Y'-~2pxzmKzYv_˵Xr* uS[WH''}c2jPhfaFX%hierlearning/domain/wargus/source_maps/map_5.pud.gz0000664000175000017500000000042612205065450022637 0ustar nevillenevilleJ0Z0=ĸi,i݄]DzQO7e"(?Phkh*.]i/2~@Jo^(n^t^β8[|inlʔ9i\fVuKt0Bķ\zR'{->.lmd7mFTǼMvh.اern4k%hierlearning/domain/wargus/source_maps/map_27.pud.gz0000664000175000017500000000043112205065450022717 0ustar nevillenevilleAJ@n\HEA @;CɴLb+ F"q:RM!CG&LHsD4D[Fgq!ÞkW-O7Gad:v3m/ a_wS4>GPDf~BIҌh1$oZlj'me暧um̗3utqjx uNvՙo~y7g0HO]Ton󘞲ePQ ["ϣ$Ɵ)=֪g< lMZuSlu$dHc:q$CiDr^8x%hierlearning/domain/wargus/source_maps/map_83.pud.gz0000664000175000017500000000042612205065450022725 0ustar nevillenevilleJ0Zw6D=b\B4[MZ7HQ/sT}Oc" BD4n$ݢGډ~,xڗc=-ĘߦzR?ߦg{<sU˳pj+:7*!EDŗ.75eLOe=d{l-[7WVpʥUdkغ]E8O<|Zq=oOy i@<, P}',%hierlearning/domain/wargus/source_maps/map_66.pud.gz0000664000175000017500000000043212205065450022723 0ustar nevillenevilleAK0C zի?@K(fI&x)E_d"%y_(/%Ph[TODcNQwqңkDHi?rc;{z~CS'_d<9pj?[tިMa_^Ȕ1=i'V魒45Mpȶ_Y!S.#[l[.S"CM M}ǤC=sRr %hierlearning/domain/wargus/source_maps/map_8.pud.gz0000664000175000017500000000041612205065450022641 0ustar nevillenevilleJ0"@K(fI&x)Ӌ9)i!IG襤- 'lDAt>ӽʍl|9{Z׷9|}c2f^KY1KBCj]?TWD]UfI[NMe0NoA ]{L938z!櫥^<* ]#gN/Tc]Otzt y T%hierlearning/domain/wargus/source_maps/map_67.pud.gz0000664000175000017500000000043212205065450022724 0ustar nevillenevilleJ@kR@|q !iLb+ U-UoSXogTDtQ;!0.. ;aJqD62s><'/a-.3Nй厌ZWh`Y9MuCBs.7ȋ![,-[gٚ72Smf k zL:ky\+ 1u?et%2>-Pڢ +ih%hierlearning/domain/wargus/source_maps/map_69.pud.gz0000664000175000017500000000041712205065450022731 0ustar nevillenevilleAJ@gR]ł.<8v0iәĶЍHP7*Um]{,J1~ 'd3LHR sHDSة|ӵc& Yf~yz[֜ݼˇ;Jx:͕vuƫY:8TT'D;o~7?#/![6ߦloṟA:^8!H& ] HjޫF-dmj%hierlearning/domain/wargus/source_maps/map_63.pud.gz0000664000175000017500000000043012205065450022716 0ustar nevillenevilleJ09p %Hդuw37:>W)Ye"# ^@ʡ&ږׅHS6 _tJYNCl:9;h`WOj$46#Jh`]Yy-:eK\r uXrlAYdkKN9iNŖcU:Gc׊?@'~) ?QP_?l%hierlearning/domain/wargus/source_maps/map_94.pud.gz0000664000175000017500000000041612205065450022726 0ustar nevillenevilleAJ@gH.<8v0iLb+t#ԍJSX_gtMT7DSv$_~]ω\?$me?>P?Ɠ_ݩ3^Mq&HRJsREߴ6$!/4g_BqmMOٲyUSDz{KD^Z+O9B[y ZѫG:mң.ePӀf2%hierlearning/domain/wargus/source_maps/map_75.pud.gz0000664000175000017500000000043112205065450022722 0ustar nevillenevilleAJ0V8 8 %ԤuFp#Rԍ`x 4 eH6fGI(M*GDSaوhzҥ$uqrqr.x&ӱힴSV̲А|YC uMa0[tNM'sE)9L?3՛2'tU2̰߽Ụ{kC~܅a\,{kC~*yBi6Ҍu}6 ܏CJu/+%hierlearning/domain/wargus/source_maps/map_64.pud.gz0000664000175000017500000000042712205065450022725 0ustar nevillenevilleJ0 e3in )ꍎܲ\T&"r(%mID4(V_t,')Ig~}y_ל}1 NtbԆwxSR\l2+ORE ߴ~I\KF: ]<rliIѽ9V5CڑetojbU!*lC./69=ui5/%hierlearning/domain/wargus/source_maps/map_97.pud.gz0000664000175000017500000000043112205065450022726 0ustar nevillenevilleAJ@gb7.b]EOı3L'F$-U+y*Oa}R4!3 ?a6@R]ODS٩ho6ХiSXfy~z]6ݼ7;Id:7uMƫY:ޕ@RJl3)|+~H,_)3՛wo}ЄJ/mu.T8fH.þGIZ>J.!֗X#aeE~ۡ DIi?%hierlearning/domain/wargus/source_maps/map_77.pud.gz0000664000175000017500000000043112205065450022724 0ustar nevillenevilleOJ@פn 3xq !iǙVp#ԅzo)_gQ̪|#&Vh&݉0>]i'>HJ#W|,jݾ?ԋ{1lVWK7i'IP]^JCʜx*|%YD8J.jn1s2!S΍׶W.v""<';Ahk9 elb'۱cUm2]z@}Zԣ. :d%hierlearning/domain/wargus/source_maps/map_26.pud.gz0000664000175000017500000000043212205065450022717 0ustar nevillenevilleAJ07"Ƞ.<'DnDQ<yfdU,̐SV> 9{M9bu~pvrny$o=i=]e%5%р`Uy+:O뗤ra2?QczC9[ Ij'cE$MVM:AȢ]MVRmu(dH)ɸK٤m%xY%hierlearning/domain/wargus/source_maps/map_47.pud.gz0000664000175000017500000000041612205065450022724 0ustar nevillenevilleJ0clLE]gĸi.ioDz2|[y e"?%m}U}"j?a*Ѫx!i&,;߽<-o;.n^OmNI;&" moz$itN65{9VZd(5G_Byz3[?އ\p,[!HKic)ɽ7E3A>BCh{ RD4:׌̟?&\̶O/]徯v}|bԆħmlj +rE ߴ~\QJF ]<zlد*qѽy{#H+vYF cU:OUl/ f_&OG? 4F%hierlearning/domain/wargus/source_maps/map_1.pud.gz0000664000175000017500000000042312205065450022630 0ustar nevillenevilleJ0[*;|1.D-K[7HQ/*S| 9(z| %^BmuMT8bOAV?^j3-m6M3d)ǕF%hierlearning/domain/wargus/source_maps/map_81.pud.gz0000664000175000017500000000043212205065450022720 0ustar nevillenevilleAJ@g .<8v0i$܈un7/oӻׇft}/ GNx5NBC[2Jj]#-Uuu7[t/I3]2?QczC;[ -gٚ`60cO9"[3ƛ, X^sEߴ~BQj'FL)[zXvotiroZӄsrHW'X57E| U>&i>mң>ɸ--9j%hierlearning/domain/wargus/source_maps/map_56.pud.gz0000664000175000017500000000042012205065450022717 0ustar nevillenevilleAJ@g.ZT]3H3F$+U1>gTD !f~l,͔SpЅ}&Yf~Z,ojή-m::< mdʹcٽEPio¹c\wPioyB'ǪT[6рd%庒Oƈ%hierlearning/domain/wargus/source_maps/map_23.pud.gz0000664000175000017500000000043012205065450022712 0ustar nevillenevilleOK0v/|b\B4[LZ7HQ/*S_f}rP&"l|)R m+h,mv]j/r>OJ#߾\8v0q&| ꋖ F\pMOz ~9CK4-s}LDCi/D&snOwڊҮ'xK5y}{=^_ vI6m(dbST<#l+c\*߸)^۠ICe(|sSMڞ̈́OeӍi&Bm?Oj٬^MM^79&G7|QPD%hierlearning/domain/wargus/source_maps/map_52.pud.gz0000664000175000017500000000042512205065450022720 0ustar nevillenevilleJ0𿭂P=_@K(fI&x)΋m}$D$~P>J mh.옍Frn?"܄vo/ezz~䉯oSδoSV,ؐ}Zوmmie-:eKJr~U67UJOz4 WjHq׆9rHȧOZk5XU.uzX!4<RAWJӝ^h%hierlearning/domain/wargus/source_maps/map_22.pud.gz0000664000175000017500000000041612205065450022715 0ustar nevillenevilleAJ@gR\TAA @;Cɴq&-tS$ *Qm=9JETH6$i,+߾>n[.nOc͝F62 meH]Cz/[t_M,̻3ԛ2lU# "HNu5dRF2'4Jӵo3o%hierlearning/domain/wargus/source_maps/map_41.pud.gz0000664000175000017500000000041612205065450022716 0ustar nevillenevilleAK0ox=~0-DKm."ezQz™L\=|^JJT"Z {FDǻ[F e9OI*7a?lxf˩ݟ)K)+VIhHm3ZVbxN#8TVSE ߴ~I\^Hˏblz3].?L2A:^[ sy5~B#j>wf=ExUP n%hierlearning/domain/wargus/source_maps/map_70.pud.gz0000664000175000017500000000042712205065450022722 0ustar nevilleneville1K@.ųw\{%HPOx9A %TVh&a+Y8CR1=<=,.o l3m櫔,NHmx#lwCр`U]8-:7eCRr}ʗ'.in˔9mk{16gZa\_INq=Z6mk"]SdtJJҐFtu%hierlearning/domain/wargus/source_maps/map_0.pud.gz0000664000175000017500000000043012205065450022625 0ustar nevillenevilleAJ@Έ.  3xq !iǙVp#ԍJxG)Ray)G&ͥ;FZa7ڣkD~"ʵo/VwALxk6'۴,,TicQ<;&lk rK#EDc%fe>d|hM*uPWF J!ԧ.cU_li;T3-MiD|鐲p %hierlearning/domain/wargus/source_maps/map_65.pud.gz0000664000175000017500000000043012205065450022720 0ustar nevillenevilleJ0Z{1@K(fveHQ/:>ՃOrP&Cr(}4V7:"lDItwi,'1Igyy~[5^=}p2+e&e4 ħ8%ls+S|E ~I\^HӏF ][L55AZ<7߁ոaeto* ]"H\bz !etnjC]ѭ?U7%hierlearning/domain/wargus/source_maps/map_72.pud.gz0000664000175000017500000000042412205065450022721 0ustar nevillenevilleAJ@gbA.<@qH3FJP7*U<YT*"]/fy<0}"*ND]ω?$mˆeż^?H&ӱ_=n2^͒X]Y$.h`Sy}+:`uK\s~%CU)[rX@)Ɏ1SAڒ&;z-Zdjl.u\>Q"RH`%hierlearning/domain/wargus/source_maps/map_18.pud.gz0000664000175000017500000000043112205065450022717 0ustar nevillenevilleAJ@gb).<8v0i$܈uzo)T&OC IuS=")NDGD6eeח}c3}z~ԗ_\2^񶌇ZWh`UyKGiuKRf0Lץ}Toh󘞲EƎeHD&X LySi3 & :C<D3VDzM:r\ȱ#{g.ӭ k%hierlearning/domain/wargus/source_maps/map_32.pud.gz0000664000175000017500000000042712205065450022720 0ustar nevillenevilleAK0V/;l? xJjҺ ^Dz1+9L&"my)IK%2D4VNUDINs&H0`}vrqΞx xWW&YlhxWCIkuvDC)#Xi OlS*xrı*9t5dsjnb9eC9"Q!ԝµ>zQ!ԝun"9S5婼ЀP"VM"Rԋwxvf9L&2r\wH(%H[_Wj&žQ١+e9H*7d??>Ɠ]i'ߤf!}Y#r2|%sE)9L?mq幍u*Xzj Aeٚ%\4C4YD hU:C4ZƵzfcMvO~рV(Sn IW%hierlearning/domain/wargus/source_maps/map_80.pud.gz0000664000175000017500000000042412205065450022720 0ustar nevillenevilleAJ0g .<'DnDQySY(#d]$#֗%GTE}]ϙ#mˆeM];Lc|jņeݗ4VDk˚aY(JqPĖ)=ergez+Q$͔V8wCMLiySN?Yt.jƶ>1٤-:u@; e%˅S%hierlearning/domain/wargus/source_maps/map_35.pud.gz0000664000175000017500000000042412205065450022720 0ustar nevillenevilleJ0cv^gĸin7"E1<˅"委@MXSv"/zsϙ?$̷/v~*zņe<4gD[›=5VGp~ 2mU>cٽ&XN•c'XN|^Α"ۤ4C[6˞G2>7g>\%hierlearning/domain/wargus/source_maps/readme.txt0000664000175000017500000000006612205065450022503 0ustar nevillenevilleSpecs: 20 x 20 1 forest, 3 trees 1 goldmine 1 peasant hierlearning/domain/wargus/source_maps/map_46.pud.gz0000664000175000017500000000042012205065450022716 0ustar nevillenevilleAJ@gvQD |gı3Lg[HP7*U]zOa}E"߿xC IʫŒh*]ω\? m„e뗧m}|xfoO7Ilh'PRRl2REߴHBk˯-!?[f6ߦl<ʅcٽt41p) {L{Ǜ,'t I{X.\ԧ}ۻ%hierlearning/domain/wargus/source_maps/map_90.pud.gz0000664000175000017500000000042012205065450022715 0ustar nevillenevilleJ0VAJ3b\B4]L[7ިL}֧p;˅2ˀڴқhkovh ;*+8M=`}rqqMy ӿ GU?i=_dgq| QD=EmV?j?_(5g<+ry!׌%Q}.VBIL:K 6GL6h.I{ٕ\(t!%hierlearning/domain/wargus/source_maps/map_62.pud.gz0000664000175000017500000000042512205065450022721 0ustar nevillenevilleJ0Vn E=~q %l5i/"E{[2al;hBSr m}]?a*μwإ+9M|6k}}hOܗh࿞/2^Fh`QSx+:`uKRs%>7eJO٪c=cٽl@&ɽJ.s<d%f+]7E0h{~PB="J"tD;xHWڋa$x曗m#}xx#LǾ;fN{9bC+6x<ԪDuu0[toM,qPĔ)=i\fVu=3'dpiw~䝓 *^~BgȲ];\cS2٦]q)ꆷO%hierlearning/domain/wargus/source_maps/map_30.pud.gz0000664000175000017500000000043212205065450022712 0ustar nevillenevilleOJ1ό  CdҎɌFdP7Z0SX,H6݄{d0YLsU"KwVD;7Aڱ{puvqyrxF[>iXPfCZ# Um^?Q/—8(\Rod4um&347f/,s<<7ftEj/ o9d<=alSA Pf%hierlearning/domain/wargus/source_maps/map_42.pud.gz0000664000175000017500000000043112205065450022714 0ustar nevillenevilleJ0((zgĸin7"EQ|7ƧXD$psPBmyS!ͤ;VD{Oډ~,x귗}-cxz~coLgS?ݦ';<8*1Q`[9u-:oTK\0J/\Toj4E-V-jBSf>8MV8Αnfլ&Vm)OxӀ.y<]Zԣ! t[-%hierlearning/domain/wargus/source_maps/map_33.pud.gz0000664000175000017500000000042012205065450022712 0ustar nevillenevilleJ0a @K(f˒MRTsP&"^d>CB h[{3$Yhgy.iG,1Qɹ>CPz rz +FqܧKEH0}3}x_N|r eX>WJVՅW[9,?PJg_yv..m .TMud6<)K9]%hierlearning/domain/wargus/wargus.cc0000664000175000017500000005107212205065450020007 0ustar nevilleneville /************************************************* WARGUS DOMAIN Neville Mehta **************************************************/ #include #include "wargus.h" Wargus_State::Wargus_State (const unsigned& num_agents) : num_global_variables(2), effective_range(4), goldmine_width(3), townhall_width(4), num_resources(2) { peasants.resize(num_agents); agent_region_gold.resize(num_agents, NA); agent_region_wood.resize(num_agents, NA); agent_region_townhall.resize(num_agents, NA); } vector Wargus_State::variables () const { vector state_variable_tokens = tokenize("agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", ", "); vector state_variables; for (const auto& state_variable_token : state_variable_tokens) state_variables.push_back(variable_index(state_variable_token)); return state_variables; } // Variable name to integer index int Wargus_State::variable_index (const string& variable_name) const { // Agent-dependent variables are positively indexed; global variables are negatively indexed if (variable_name == "agent_x") return 0; if (variable_name == "agent_y") return 1; if (variable_name == "agent_resource") return 2; if (variable_name == "agent_region_gold") return 3; if (variable_name == "agent_region_wood") return 4; if (variable_name == "agent_region_townhall") return 5; // Agent variables are named "agent#_" if (variable_name.substr(0, 5) == "agent") { unsigned agent = from_string(variable_name.substr(5, variable_name.find_first_of("_") - 5)); if (agent >= num_agents()) throw HierException(__FILE__, __LINE__, "Not a valid agent id."); if (variable_name.substr(variable_name.find_first_of("_")) == "_x") return num_agent_variables() * agent; else if (variable_name.substr(variable_name.find_first_of("_")) == "_y") return num_agent_variables() * agent + 1; else if (variable_name.substr(variable_name.find_first_of("_")) == "_resource") return num_agent_variables() * agent + 2; else if (variable_name.substr(variable_name.find_first_of("_")) == "_region_gold") return num_agent_variables() * agent + 3; else if (variable_name.substr(variable_name.find_first_of("_")) == "_region_wood") return num_agent_variables() * agent + 4; else if (variable_name.substr(variable_name.find_first_of("_")) == "_region_townhall") return num_agent_variables() * agent + 5; } else if (variable_name == "requisite_gold") return -1; else if (variable_name == "requisite_wood") return -2; throw HierException(__FILE__, __LINE__, "Unknown variable \"" + variable_name + "\"."); } string Wargus_State::variable_name (const int& variable_index) const { if (variable_index >= 0) { switch (variable_index % num_agent_variables()) { case 0: return "agent_x"; case 1: return "agent_y"; case 2: return "agent_resource"; case 3: return "agent_region_gold"; case 4: return "agent_region_wood"; case 5: return "agent_region_townhall"; default: throw HierException(__FILE__, __LINE__, "Unknown variable #" + to_string(variable_index) + "."); } } else switch (abs(variable_index)) { case 1: return "requisite_gold"; case 2: return "requisite_wood"; default: throw HierException(__FILE__, __LINE__, "Unknown variable #" + to_string(variable_index) + "."); } } // Arity of a variable int Wargus_State::variable_size (const int& variable_index) const { if (variable_index >= 0) { switch (variable_index % num_agent_variables()) { case 0: // x return map_size().x; case 1: // y return map_size().y; case 2: // resource return num_resources + 1; case 3: // gold_in_region case 4: // wood_in_region case 5: // townhall_in_region return 2; default: throw HierException(__FILE__, __LINE__, "Unknown variable #" + to_string(variable_index) + "."); } } else return 2; // Requisite gold & wood } int Wargus_State::variable (const int& variable_index) const { if (variable_index >= 0) { unsigned agent = variable_index / num_agent_variables(); switch (variable_index % num_agent_variables()) { case 0: return peasants[agent].coord.x; case 1: return peasants[agent].coord.y; case 2: return peasants[agent].resource; case 3: return agent_region_gold[agent]; case 4: return agent_region_wood[agent]; case 5: return agent_region_townhall[agent]; default: throw HierException(__FILE__, __LINE__, "Unknown variable #" + to_string(variable_index) + "."); } } else if (variable_index == -1) return requisite_resources.gold; else if (variable_index == -2) return requisite_resources.wood; else throw HierException(__FILE__, __LINE__, "Unknown variable #" + to_string(variable_index) + "."); } int& Wargus_State::variable (const int& variable_index) { if (variable_index >= 0) { unsigned agent = variable_index / num_agent_variables(); switch (variable_index % num_agent_variables()) { case 0: return peasants[agent].coord.x; case 1: return peasants[agent].coord.y; case 2: return peasants[agent].resource; case 3: return agent_region_gold[agent]; case 4: return agent_region_wood[agent]; case 5: return agent_region_townhall[agent]; default: throw HierException(__FILE__, __LINE__, "Illegal variable assignment."); } } else if (variable_index == -1) return requisite_resources.gold; else if (variable_index == -2) return requisite_resources.wood; else throw HierException(__FILE__, __LINE__, "Unknown variable #" + to_string(variable_index) + "."); } map Wargus_State::variables_mapper () const { map var_map; for (unsigned i = 0; i < num_agent_variables(); ++i) var_map[i] = variable(i); for (int i = 1; i <= (int)num_global_variables; ++i) var_map[-i] = variable(-i); return var_map; } bool Wargus_State::action_complete (const unsigned& agent) const { return peasants[agent].status == 1; } Coordinate Wargus_State::map_size () const { return Coordinate(map_layout.num_rows(), map_layout.num_cols()); } void Wargus_State::draw_map () const { for (int y = 0; y < map_size().y; ++y) { for (int x = 0; x < map_size().x; ++x) cout << map_layout(x, y); cout << endl; } cout << endl; } pair Wargus_State::parse (string expression) const { int total = 0; // Delete spaces while (expression.find(" ") != string::npos) expression.replace(expression.find(" "), 1, ""); expression.insert(0, "+"); // Explicit sign for positive numbers at the start string::size_type start = expression.find_first_not_of("+-", 0); string::size_type end = expression.find_first_of("+-", start); while (start != string::npos || end != string::npos) { istringstream token(expression.substr(start, end - start)); int value; if (!(token >> value)) // 'token' is not an integer { if (token.str() == "num_agent_variables") value = num_agent_variables(); else if (token.str() == "NL") value = NL; else if (token.str() == "Gold") value = Gold; else if (token.str() == "Wood") value = Wood; else if (token.str() == "Oil") value = Oil; else return make_pair(false, 0); } total += (expression[start-1] == '+') ? value : -value; start = expression.find_first_not_of("+-", end); end = expression.find_first_of("+-", start); } return make_pair(true, total); } void Wargus_State::read (istream& in) { for (unsigned agent = 0; agent < num_agents(); ++agent) { if (in >> peasants[agent].coord) { in >> peasants[agent].resource; in >> agent_region_gold[agent]; in >> agent_region_wood[agent]; in >> agent_region_townhall[agent]; } else throw HierException(__FILE__, __LINE__, "Bad state read."); } in >> requisite_resources.gold; in >> requisite_resources.wood; } string Wargus_State::print () const { ostringstream out; for (unsigned agent = 0; agent < num_agents(); ++agent) { if (agent > 0) out << " "; out << "("; if (peasants[agent].coord.x != NA) out << peasants[agent].coord.x; else out << "*"; out << ","; if (peasants[agent].coord.y != NA) out << peasants[agent].coord.y; else out << "*"; out << ")"; if (peasants[agent].resource != NA) out << " " << peasants[agent].resource; else out << " *"; if (agent_region_gold[agent] != NA) out << " " << agent_region_gold[agent]; else out << " *"; if (agent_region_wood[agent] != NA) out << " " << agent_region_wood[agent]; else out << " *"; if (agent_region_townhall[agent] != NA) out << " " << agent_region_townhall[agent]; else out << " *"; } if (requisite_resources.gold != NA) out << " " << requisite_resources.gold; else out << " *"; if (requisite_resources.wood != NA) out << " " << requisite_resources.wood; else out << " *"; return out.str(); } //***************************************************************************************** Wargus::Wargus (const string& name, ClientSocket* const socket, const string& source_map_name, const unsigned& num_source_maps, const Wargus_State::Resources& source_resource_quotas, const string& target_map_name, const unsigned& num_target_maps, const Wargus_State::Resources& target_resource_quotas, const int& speed, const int& cycles) : MDP(name, new Wargus_State(0)), socket(socket), source_map_name(source_map_name), num_source_maps(num_source_maps), source_resource_quotas(source_resource_quotas), target_map_name(target_map_name), num_target_maps(num_target_maps), target_resource_quotas(target_resource_quotas), target_map_initialized(false), reward_default(-1), reward_deposit(-1) { // Check that all source maps exist for (unsigned m = 0; m < num_source_maps; ++m) { if (!entity_exists("../other/stratagus/data/" + source_map_name + to_string(m) + ".pud.gz")) throw HierException(__FILE__, __LINE__, "Map ../other/stratagus/" + source_map_name + to_string(m) + ".pud.gz does not exist."); } // Check that all target maps exist for (unsigned m = 0; m < num_target_maps; ++m) { if (!entity_exists("../other/stratagus/data/" + target_map_name + to_string(m) + ".pud.gz")) throw HierException(__FILE__, __LINE__, "Map ../other/stratagus/" + target_map_name + to_string(m) + ".pud.gz does not exist."); } set_speed(speed, cycles); } valarray Wargus::reward () const { valarray current_reward(0.0, _reward.size()); for (unsigned agent = 0; agent < _reward.size(); ++agent) if (state().action_complete(agent)) current_reward[agent] = _reward[agent]; return current_reward; } void Wargus::initialize (const bool& target) { if (target) { if (target_map_initialized) reset(); else { load_map(target_map_name + to_string(rand_int(num_target_maps)) + ".pud.gz"); state().resource_quotas = target_resource_quotas; target_map_initialized = true; } } else { load_map(source_map_name + to_string(rand_int(num_source_maps)) + ".pud.gz"); state().resource_quotas = source_resource_quotas; } socket->send("l g\n"); read_socket(true); string::size_type index = 0; player_id = tokenizer("player-id", index); Coordinate map_size; map_size.x = tokenizer("width", index); map_size.y = tokenizer("length", index); state().map_layout.resize(map_size.x, map_size.y); state_update(); _duration.resize(state().num_agents()); _reward.resize(state().num_agents()); } void Wargus::read_socket (const bool parentheses_matching, const unsigned& num_lines) { if (parentheses_matching) { unsigned open_parens = 0, close_parens = 0; string buffer; response.clear(); do { if (!socket->receive(buffer)) throw HierException(__FILE__, __LINE__, "Connection closed by Stratagus server."); for (const auto& c : buffer) { if (c == '(') ++open_parens; else if (c == ')') ++close_parens; } response += buffer; } while (open_parens != close_parens || response[response.size()-1] != '\n'); } else { unsigned received_lines = 0; string buffer; response.clear(); do { if (!socket->receive(buffer)) throw HierException(__FILE__, __LINE__, "Connection closed by Stratagus server."); for (const auto& c : buffer) if (c == '\n') ++received_lines; response += buffer; } while (received_lines < num_lines); } } int Wargus::tokenizer (const string& search_str, string::size_type& index) const { string::size_type start = response.find_first_of("0123456789", response.find(search_str, index)); index = response.find_first_not_of("0123456789", start); return from_string(response.substr(start, index - start)); } void Wargus::load_map (const string& map_filename) { socket->send("m " + map_filename + "\n"); read_socket(false); } void Wargus::read_map () { socket->send("l m\n"); read_socket(true); string::size_type start = response.find_first_not_of("()\n"); for (int y = 0; y < state().map_size().y; ++y) { for (int x = 0; x < state().map_size().x; ++x) state().map_layout(x, y) = response[start + x]; start = response.find_first_not_of("()\n", start + state().map_size().x); } } void Wargus::state_update () { read_map(); vector peasant_previous_state = state().peasants; state().peasants.clear(); state().townhalls.clear(); state().goldmines.clear(); socket->send("l s\n"); read_socket(true); // cout << response << endl; // Causes drastic (stochastic) slowdown in Windows when attempting to run at full speed string::size_type index = 0; while (response.find(" . ", index) != string::npos) { index = response.find_last_of('(', response.find(" . ", index)); Wargus_State::Unit unit; unit.id = tokenizer("", index); if (tokenizer("player-id", index) == player_id) // Our units only { unit.type = tokenizer("type", index); unit.coord.x = tokenizer("loc", index); unit.coord.y = tokenizer("", index); unit.hit_points = tokenizer("hp", index); unit.value = tokenizer("r-amt", index); unit.status = tokenizer("status", index); if (unit.type == 2) { if (unit.value > 0) { if (unit.status == 1 || unit.status == 7) unit.resource = (Wargus_State::Resource)tokenizer("status-args", index); else unit.resource = peasant_previous_state[state().peasants.size()].resource; } else unit.resource = Wargus_State::NL; } switch (unit.type) { case 2: state().peasants.push_back(unit); break; case 74: state().townhalls.push_back(unit); break; case 92: state().goldmines.push_back(unit); break; default:; } } } // Get amount of stored gold & wood state().resources_gathered.gold = tokenizer(":gold", index); state().resources_gathered.wood = tokenizer(":wood", index); // Update internal variables state().agent_region_gold.clear(); state().agent_region_wood.clear(); state().agent_region_townhall.clear(); state().agent_region_gold.resize(state().num_agents(), Wargus_State::NA); state().agent_region_wood.resize(state().num_agents(), Wargus_State::NA); state().agent_region_townhall.resize(state().num_agents(), Wargus_State::NA); for (unsigned agent = 0; agent < state().num_agents(); ++agent) { Wargus_State::Region agent_region; agent_region.coord = state().peasants[agent].coord; agent_region.left_up_offset = Coordinate(min(agent_region.coord.x, state().effective_range), min(agent_region.coord.y, state().effective_range)); agent_region.right_down_offset = Coordinate(min(state().map_size().x - 1 - agent_region.coord.x, state().effective_range), min(state().map_size().y - 1 - agent_region.coord.y, state().effective_range)); // Checking for gold (goldmines[g].coord stores the coord of the top left corner of every mine) for (const auto& goldmine : state().goldmines) if ((goldmine.coord.x <= agent_region.coord.x ? agent_region.coord.x - goldmine.coord.x < agent_region.left_up_offset.x + state().goldmine_width : goldmine.coord.x - agent_region.coord.x <= agent_region.right_down_offset.x) && (goldmine.coord.y <= agent_region.coord.y ? agent_region.coord.y - goldmine.coord.y < agent_region.left_up_offset.y + state().goldmine_width : goldmine.coord.y - agent_region.coord.y <= agent_region.right_down_offset.y) && goldmine.value > 0) { state().agent_region_gold[agent] = 1; break; } if (state().agent_region_gold[agent] == Wargus_State::NA) state().agent_region_gold[agent] = 0; // Checking for wood for (int y = agent_region.coord.y - agent_region.left_up_offset.y; y <= agent_region.coord.y + agent_region.right_down_offset.y; ++y) for (int x = agent_region.coord.x - agent_region.left_up_offset.x; x <= agent_region.coord.x + agent_region.right_down_offset.x; ++x) if (state().map_layout(x, y) == 'T') // Tree found { state().agent_region_wood[agent] = 1; break; } if (state().agent_region_wood[agent] == Wargus_State::NA) state().agent_region_wood[agent] = 0; // Checking for town halls (townhall[g].coord stores the coord of the top left corner of every town hall) for (const auto& townhall : state().townhalls) if ((townhall.coord.x <= agent_region.coord.x ? agent_region.coord.x - townhall.coord.x < agent_region.left_up_offset.x + state().townhall_width : townhall.coord.x - agent_region.coord.x <= agent_region.right_down_offset.x) && (townhall.coord.y <= agent_region.coord.y ? agent_region.coord.y - townhall.coord.y < agent_region.left_up_offset.y + state().townhall_width : townhall.coord.y - agent_region.coord.y <= agent_region.right_down_offset.y)) { state().agent_region_townhall[agent] = 1; break; } if (state().agent_region_townhall[agent] == Wargus_State::NA) state().agent_region_townhall[agent] = 0; } state().requisite_resources.gold = state().resources_gathered.gold < state().resource_quotas.gold ? 0 : 1; state().requisite_resources.wood = state().resources_gathered.wood < state().resource_quotas.wood ? 0 : 1; } void Wargus::reset () { socket->send("r\n"); read_socket(false); } void Wargus::quit () { socket->send("q\n"); while (socket->receive(response)); } void Wargus::kill () { socket->send("k\n"); } void Wargus::set_speed (const int& game_speed, const int& cycles_per_video_update) { ostringstream out_stream; out_stream << "s " << game_speed << endl; socket->send(out_stream.str()); read_socket(false); out_stream.str(""); out_stream << "v " << cycles_per_video_update << endl; socket->send(out_stream.str()); read_socket(false); } unsigned Wargus::get_cycle () { socket->send("x\n"); read_socket(false); return from_string(response); } void Wargus::transition (const unsigned& num_cycles) const { ostringstream out_stream; out_stream << "t " << num_cycles << endl; socket->send(out_stream.str()); } void Wargus::noop (const unsigned& unit_id) const { ostringstream out_stream; out_stream << "c " << unit_id << " 0\n"; socket->send(out_stream.str()); } void Wargus::stop (const unsigned& unit_id) const { ostringstream out_stream; out_stream << "c " << unit_id << " 1\n"; socket->send(out_stream.str()); } void Wargus::move (const unsigned& unit_id, const Coordinate& coord) const { ostringstream out_stream; out_stream << "c " << unit_id << " 2 " << coord.x << " " << coord.y << endl; socket->send(out_stream.str()); } bool Wargus::harvest (const unsigned& unit_id, const unsigned& resource) { ostringstream out_stream; out_stream << "c " << unit_id << " 6 " << resource << endl; socket->send(out_stream.str()); read_socket(false); if (response == "OK\n") return true; else if (response == "Unable to find gold within range.\n" || response == "Unable to find wood within range.\n") return false; else throw HierException(__FILE__, __LINE__, "Unknown response from Stratagus."); } bool Wargus::deposit (const unsigned& unit_id) { ostringstream out_stream; out_stream << "c " << unit_id << " 7\n"; socket->send(out_stream.str()); read_socket(false); if (response == "OK\n") return true; else if (response == "Unit has no resources to return.\n" || response == "Unable to find location to return resource within range.\n") return false; else throw HierException(__FILE__, __LINE__, "Unknown response from Stratagus."); } void Wargus::process (const vector& actions) { bool all_actions_in_progress = true; while (all_actions_in_progress) { transition(30); state_update(); for (unsigned agent = 0; agent < state().num_agents(); ++agent) { if (!state().action_complete(agent)) { _duration[agent]++; _reward[agent] += reward_default; } else all_actions_in_progress = false; } } for (auto da_itr = depositing_agents.begin(); da_itr != depositing_agents.end(); ) { if (state().action_complete(*da_itr)) da_itr = depositing_agents.erase(da_itr); else ++da_itr; } } Wargus::~Wargus () { set_speed(100, 1); quit(); // kill(); delete socket; } hierlearning/domain/wargus/gen_maps.py0000664000175000017500000000023012205065450020321 0ustar nevilleneville#!/usr/bin/python # Makes multiple Wargus maps import os for m in range(100): os.system("rgmap 20 20 3 15 2 3 -o maps/map_" + str(m) + ".pud.gz") hierlearning/domain/wargus/hi-mat_hierarchy.cc0000664000175000017500000001317412205065450021715 0ustar nevilleneville//***** Termination functions *****// bool GotoGold_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(agent * state.num_agent_variables() + state.variable_index("agent_region_gold")) == 1; } bool GotoWood_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(agent * state.num_agent_variables() + state.variable_index("agent_region_wood")) == 1; } bool GotoTown_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(agent * state.num_agent_variables() + state.variable_index("agent_region_townhall")) == 1; } bool GetGold_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(agent * state.num_agent_variables() + state.variable_index("agent_resource")) == 1; } bool GetWood_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(agent * state.num_agent_variables() + state.variable_index("agent_resource")) == 2; } bool Put_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(agent * state.num_agent_variables() + state.variable_index("agent_resource")) == 0; } bool HarvestGold_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(state.variable_index("requisite_gold")) == 1; } bool HarvestWood_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(state.variable_index("requisite_wood")) == 1; } bool WargusRoot_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(state.variable_index("requisite_gold")) == 1 && state.variable(state.variable_index("requisite_wood")) == 1; } //***** Constructor *****// { vector move; Task* mine_gold = 0, * chop_wood = 0, * deposit = 0; const Wargus_State& state = static_cast(mdp.state()); move.resize(state.regions.size()); for (unsigned site = 0; site < move.size(); ++site) { move[site].link = new PrimTask(string("Goto") + "_" + to_string(state.regions.data()[site].coord.x) + "_" + to_string(state.regions.data()[site].coord.y), "agent_x, agent_y", Wargus_Goto_MDP::Move + state.regions.data()[site].coord.x + state.regions.data()[site].coord.y * state.map_size().x, PrimTask::Tabular, mdp); } for (unsigned m = 0; m < move.size(); ++m) Task_list.push_back(move[m].link); mine_gold = new PrimTask("MineGold", "agent_resource, agent_region_gold, requisite_gold", Wargus_Goto_MDP::MineGold, PrimTask::Tabular, mdp); Task_list.push_back(mine_gold); chop_wood = new PrimTask("ChopWood", "agent_resource, agent_region_wood, requisite_wood", Wargus_Goto_MDP::ChopWood, PrimTask::Tabular, mdp); Task_list.push_back(chop_wood); deposit = new PrimTask("Deposit", "agent_resource, agent_region_townhall", Wargus_Goto_MDP::Deposit, PrimTask::Tabular, mdp); Task_list.push_back(deposit); vector subtasks(move); Task* GotoGold = new CompositeTask("GotoGold", "", "agent_x, agent_y", subtasks, &GotoGold_termination, Expression(), mdp); Task_list.push_back(GotoGold); Task* GotoWood = new CompositeTask("GotoWood", "", "agent_x, agent_y", subtasks, &GotoWood_termination, Expression(), mdp); Task_list.push_back(GotoWood); Task* GotoTown = new CompositeTask("GotoTown", "", "agent_x, agent_y", subtasks, &GotoTown_termination, Expression(), mdp); Task_list.push_back(GotoTown); subtasks.clear(); subtasks.push_back(GotoGold); subtasks.push_back(mine_gold); Task* GetGold = new CompositeTask("GetGold", "", "agent_x, agent_y, agent_resource, agent_region_gold, requisite_gold", subtasks, &GetGold_termination, Expression(), mdp); Task_list.push_back(GetGold); subtasks.clear(); subtasks.push_back(GotoTown); subtasks.push_back(deposit); Task* PutGold = new CompositeTask("PutGold", "", "agent_x, agent_y, agent_resource, agent_region_townhall", subtasks, &Put_termination, Expression(), mdp); Task_list.push_back(PutGold); subtasks.clear(); subtasks.push_back(GotoWood); subtasks.push_back(chop_wood); Task* GetWood = new CompositeTask("GetWood", "", "agent_x, agent_y, agent_resource, agent_region_wood, requisite_wood", subtasks, &GetWood_termination, Expression(), mdp); Task_list.push_back(GetWood); subtasks.clear(); subtasks.push_back(GotoTown); subtasks.push_back(deposit); Task* PutWood = new CompositeTask("PutWood", "", "agent_x, agent_y, agent_resource, agent_region_townhall", subtasks, &Put_termination, Expression(), mdp); Task_list.push_back(PutWood); subtasks.clear(); subtasks.push_back(GetGold); subtasks.push_back(PutGold); Task* HarvestGold = new CompositeTask("HarvestGold", "", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_townhall, requisite_gold", subtasks, &HarvestGold_termination, Expression(), mdp); Task_list.push_back(HarvestGold); subtasks.clear(); subtasks.push_back(GetWood); subtasks.push_back(PutWood); Task* HarvestWood = new CompositeTask("HarvestWood", "", "agent_x, agent_y, agent_resource, agent_region_wood, agent_region_townhall, requisite_wood", subtasks, &HarvestWood_termination, Expression(), mdp); Task_list.push_back(HarvestWood); // Root task subtasks.clear(); subtasks.push_back(HarvestGold); subtasks.push_back(HarvestWood); Task_list.push_front(new CompositeTask("Resource_Gathering_Task", "", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", subtasks, &WargusRoot_termination, Expression(), mdp)); } hierlearning/domain/wargus/client/0000775000175000017500000000000012304226346017444 5ustar nevillenevillehierlearning/domain/wargus/client/socket.h0000664000175000017500000000353412205065450021107 0ustar nevilleneville /************************************************ SOCKET CLASS Neville Mehta *************************************************/ #pragma once #if defined(_WIN32) #include #else #define SOCKET int #endif #include #include class Socket { private: unsigned* _num_references; static unsigned _num_of_sockets; static void initialize(); protected: SOCKET _socket; public: Socket(); Socket(const SOCKET&); Socket(const Socket&); Socket& operator = (const Socket& s); void send(std::string) const; bool is_readable(const long int& secs = 0, const long int& microsecs = 100000) const; // Non-blocking call to check socket bool receive(std::string&) const; // Blocking reception bool receive_timeout(std::string&, const unsigned& secs = 0, const unsigned& microsecs = 100000) const; // Non-blocking reception (with timeout) virtual ~Socket(); }; //***********************************************************************************************// class ClientSocket : public Socket { public: ClientSocket(const std::string&, const int&); virtual ~ClientSocket () {} }; //***********************************************************************************************// class ServerSocket : public Socket { public: ServerSocket(const int&, const int&, const bool non_blocking = false); Socket* accept() const; virtual ~ServerSocket () {} }; //***********************************************************************************************// class SocketException : public std::exception { private: std::string _error_message; public: SocketException (const std::string& error_message); const char* what () const throw() { return _error_message.c_str(); } ~SocketException () throw() {} }; //***********************************************************************************************// hierlearning/domain/wargus/client/socket.cc0000664000175000017500000001371312205065450021245 0ustar nevilleneville /************************************************ SOCKET CLASS Neville Mehta *************************************************/ #if !defined(_WIN32) #include #include #include #include #include #include #include #include #define INVALID_SOCKET -1 #define SOCKET_ERROR -1 #endif #include #include #include #include "socket.h" const int buffer_size = 4096; unsigned Socket::_num_of_sockets = 0; Socket::Socket () { initialize(); _socket = socket(AF_INET, SOCK_STREAM, 0); // TCP socket if (_socket == INVALID_SOCKET) throw SocketException("Socket::Socket"); ++_num_of_sockets; _num_references = new unsigned(1); } Socket::Socket (const SOCKET& s) : _socket(s) { initialize(); _num_references = new unsigned(1); } Socket::Socket (const Socket& s) : _socket(s._socket) { _num_references = s._num_references; (*_num_references)++; } Socket& Socket::operator = (const Socket& s) { _socket = s._socket; _num_references = s._num_references; (*_num_references)++; return *this; } #if defined(_WIN32) void Socket::initialize () { if (!_num_of_sockets) { WSADATA wsa_data; if (WSAStartup(MAKEWORD(2, 2), &wsa_data)) throw SocketException("Socket::initialize"); } } #else void Socket::initialize () {} #endif void Socket::send (std::string message) const { do { int sent_bytes = ::send(_socket, message.c_str(), int(message.size()), 0); if (sent_bytes == SOCKET_ERROR) throw SocketException("Socket::send"); message.erase(0, sent_bytes); } while (message.size()); } bool Socket::is_readable (const long int& secs, const long int& microsecs) const { fd_set fds; FD_ZERO(&fds); FD_SET(_socket, &fds); timeval tval = {secs, microsecs}; // Timeout in seconds & microseconds if (select(FD_SETSIZE, &fds, nullptr, nullptr, &tval) == SOCKET_ERROR) throw SocketException("Socket::is_readable"); return FD_ISSET(_socket, &fds) ? true : false; } bool Socket::receive (std::string& message) const { char buffer[buffer_size]; int recvd_bytes; unsigned long arg = 0; message.clear(); do { recvd_bytes = ::recv(_socket, buffer, buffer_size, 0); if (recvd_bytes == SOCKET_ERROR) throw SocketException("Socket::receive"); message.append(buffer, recvd_bytes); #if defined(_WIN32) } while (!ioctlsocket(_socket, FIONREAD, &arg) && arg > 0); #else } while (!ioctl(_socket, FIONREAD, &arg) && arg > 0); #endif return recvd_bytes ? true : false; // Socket open? } bool Socket::receive_timeout (std::string& message, const unsigned& secs, const unsigned& microsecs) const { char buffer[buffer_size]; int recvd_bytes = -1; message.clear(); while (is_readable(secs, microsecs)) { recvd_bytes = ::recv(_socket, buffer, buffer_size, 0); if (recvd_bytes == 0) break; if (recvd_bytes == SOCKET_ERROR) throw SocketException("Socket::receive_delay"); message.append(buffer, recvd_bytes); } return recvd_bytes ? true : false; // Socket open? } Socket::~Socket () { if (!--(*_num_references)) { delete _num_references; --_num_of_sockets; #if defined(_WIN32) closesocket(_socket); if (!_num_of_sockets) WSACleanup(); #else close(_socket); #endif } } //***********************************************************************************************// ClientSocket::ClientSocket (const std::string& server_address, const int& port) : Socket() { std::cout << "Connecting to " << server_address << ":" << port << " ... "; sockaddr_in service; memset(&service, 0, sizeof(sockaddr_in)); service.sin_family = AF_INET; service.sin_port = htons(port); if ((service.sin_addr.s_addr = inet_addr(server_address.c_str())) == INADDR_NONE) // server_address in dotted quad notation? { hostent* host_info; if ((host_info = gethostbyname(server_address.c_str())) == 0) // server_address in DNS text notation? throw SocketException("ClientSocket::ClientSocket"); service.sin_addr = *((in_addr*)host_info->h_addr); } if (connect(_socket, (sockaddr*)&service, sizeof(sockaddr))) throw SocketException("ClientSocket::ClientSocket"); int flag = 1; if (setsockopt(_socket, IPPROTO_TCP, TCP_NODELAY, (char*)&flag, sizeof(flag)) == -1) throw SocketException("ClientSocket::ClientSocket"); std::cout << "done.\n"; } //***********************************************************************************************// ServerSocket::ServerSocket (const int& port, const int& backlog, const bool non_blocking) : Socket() { sockaddr_in service; memset(&service, 0, sizeof(sockaddr_in)); service.sin_family = AF_INET; service.sin_addr.s_addr = INADDR_ANY; service.sin_port = htons(port); int opt = 1; setsockopt(_socket, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt)); if (non_blocking) { unsigned long arg = 1; #if defined(_WIN32) ioctlsocket(_socket, FIONBIO, &arg); #else ioctl(_socket, FIONBIO, &arg); // int opts = fcntl(_socket, F_GETFL); // fcntl(_socket, F_SETFL, opts | O_NONBLOCK); #endif } if (bind(_socket, (sockaddr*)&service, sizeof(sockaddr)) == SOCKET_ERROR) throw SocketException("ServerSocket::ServerSocket"); if (listen(_socket, backlog) == SOCKET_ERROR) throw SocketException("ServerSocket::ServerSocket"); } Socket* ServerSocket::accept () const { SOCKET new_socket = ::accept(_socket, nullptr, nullptr); if (new_socket == INVALID_SOCKET) { #if defined(_WIN32) if (WSAGetLastError() != WSAEWOULDBLOCK) #else if (errno != EWOULDBLOCK) #endif throw SocketException("ServerSocket::accept"); } return new Socket(new_socket); } //***********************************************************************************************// SocketException::SocketException (const std::string& error_message) { std::ostringstream error_message_stream; error_message_stream << "\nException in " << error_message << "."; #if defined(_WIN32) error_message_stream << " Socket error: " << WSAGetLastError(); #else error_message_stream << " Socket error: " << errno; #endif _error_message = error_message_stream.str(); } hierlearning/domain/wargus/wargus_goto.cc0000664000175000017500000001055112205065450021034 0ustar nevilleneville /************************************************* WARGUS DOMAIN WITH GOTO NAVIGATION Neville Mehta **************************************************/ #include #include "wargus_goto.h" vector Wargus_Goto::actions () const { vector mdp_actions; mdp_actions.push_back(MineGold); mdp_actions.push_back(ChopWood); mdp_actions.push_back(Deposit); for (const auto& site : state().regions.data()) mdp_actions.push_back(Move + site.coord.x + site.coord.y * state().map_size().x); return mdp_actions; } void Wargus_Goto::initialize (const bool& target) { Wargus::initialize(target); // Primitive actions are successful within the effective_range. // We divide the map into as many regions of (2 * Wargus_State::effective_range + 1) x (2 * Wargus_State::effective_range + 1) cells and store their centroids. Coordinate num_regions = state().map_size() / (2 * state().effective_range + 1); Coordinate remainder = state().map_size() % (2 * state().effective_range + 1); num_regions += Coordinate(remainder.x ? 1 : 0, remainder.y ? 1 : 0); state().regions.resize(num_regions.x, num_regions.y); for (int y = 0; y < num_regions.y; ++y) for (int x = 0; x < num_regions.x; ++x) { state().regions(x, y).left_up_offset = state().regions(x, y).right_down_offset = Coordinate(state().effective_range, state().effective_range); if (x == num_regions.x - 1 && remainder.x) { state().regions(x, y).left_up_offset.x = remainder.x / 2; state().regions(x, y).right_down_offset.x = remainder.x - 1 - state().regions(x, y).left_up_offset.x; } if (y == num_regions.y - 1 && remainder.y) { state().regions(x, y).left_up_offset.y = remainder.y / 2; state().regions(x, y).right_down_offset.y = remainder.y - 1 - state().regions(x, y).left_up_offset.y; } state().regions(x, y).coord = Coordinate(x * (2 * state().effective_range + 1) + state().regions(x, y).left_up_offset.x, y * (2 * state().effective_range + 1) + state().regions(x, y).left_up_offset.y); } } void Wargus_Goto::process (const vector& actions) { for (unsigned agent = 0; agent < state().num_agents(); ++agent) { if (actions[agent] == -1) continue; if (!state().action_complete(agent)) // Ongoing primitive action { _duration[agent]++; _reward[agent] += reward_default; } else { _duration[agent] = 1.0; _reward[agent] = reward_default; switch (actions[agent]) { case MineGold: if (state().num_agents() > 1 || (state().resources_gathered.gold < state().resource_quotas.gold && state().peasants[agent].resource == Wargus_State::NL)) harvest(state().peasants[agent].id, Wargus_State::Gold); break; case ChopWood: if (state().num_agents() > 1 || (state().resources_gathered.wood < state().resource_quotas.wood && state().peasants[agent].resource == Wargus_State::NL)) harvest(state().peasants[agent].id, Wargus_State::Wood); break; case Deposit: if (deposit(state().peasants[agent].id)) if ((state().peasants[agent].resource == Wargus_State::Gold && state().resources_gathered.gold < state().resource_quotas.gold) || (state().peasants[agent].resource == Wargus_State::Wood && state().resources_gathered.wood < state().resource_quotas.wood)) { _reward[agent] += reward_deposit; depositing_agents.push_back(agent); // Stratagus increments the gold/wood store immediately } break; default: move(state().peasants[agent].id, Coordinate((actions[agent] - Move) % state().map_size().x, (actions[agent] - Move) / state().map_size().x)); } } } Wargus::process(actions); } int Wargus_Goto::action_index (const string& action_name) const { if (action_name == "MineGold") return MineGold; if (action_name == "ChopWood") return ChopWood; if (action_name == "Deposit") return Deposit; if (action_name.substr(0,5) == "Goto_") { vector coords = tokenize(action_name.substr(5), "_"); return Move + from_string(coords[0]) + from_string(coords[1]) * state().map_size().x; } throw HierException(__FILE__, __LINE__, "Unknown action name."); } string Wargus_Goto::print_action (const int& action) const { switch (action) { case MineGold: return "MineGold"; case ChopWood: return "ChopWood"; case Deposit: return "Deposit"; default: return "Goto_" + to_string((action - Move) % state().map_size().x) + "_" + to_string((action - Move) / state().map_size().x); } } hierlearning/domain/wargus/wargus_nsew.h0000664000175000017500000000201012205065452020673 0ustar nevilleneville /************************************************* WARGUS DOMAIN WITH NSEW NAVIGATION Neville Mehta **************************************************/ #pragma once #include "wargus.h" class Wargus_NSEW : public Wargus { public: enum Action {North, South, East, West, MineGold, ChopWood, Deposit}; Wargus_NSEW(const string& name, ClientSocket* const socket, const string& source_map_name, const unsigned& num_source_maps, const Wargus_State::Resources& source_resource_quotas, const string& target_map_name, const unsigned& num_target_maps, const Wargus_State::Resources& target_resource_quotas, const int& speed, const int& cycles) : Wargus(name, socket, source_map_name, num_source_maps, source_resource_quotas, target_map_name, num_target_maps, target_resource_quotas, speed, cycles) { Wargus::initialize(); } vector actions() const; void process(const vector& actions); int action_index(const string& action_name) const; string print_action(const int& action) const; ~Wargus_NSEW () {} }; hierlearning/domain/wargus/wargus.h0000664000175000017500000001225012213030336017636 0ustar nevilleneville /************************************************* WARGUS DOMAIN Neville Mehta **************************************************/ #pragma once #include #include #include #include #include #include #include "../../lib/common.h" #include "../../lib/matrix.h" #include "client/socket.h" #include "../mdp.h" struct Wargus_State : public State { const unsigned num_global_variables; // Number of agent-independent variables {requisite_gold, requisite_gold} const int effective_range; // Unit's effective range const int goldmine_width; // Gold mine occupies 3 x 3 cells on the map const int townhall_width; // Town hall occupies 4 x 4 cells on the map enum Resource {NA = -1, NL, Gold, Wood, Oil}; const int num_resources; struct Unit { unsigned type; unsigned id; Coordinate coord; unsigned hit_points; int resource; unsigned value; unsigned status; Unit () : type(NA), id(NA), coord(NA), hit_points(NA), resource(NA), value(NA), status(NA) {} }; vector peasants; vector goldmines; vector townhalls; matrix map_layout; struct Region { Coordinate coord; Coordinate left_up_offset; Coordinate right_down_offset; }; matrix regions; // Binary indicator variables vector agent_region_gold; vector agent_region_wood; vector agent_region_townhall; struct Resources { int gold; int wood; int oil; Resources () : gold(NA), wood(NA), oil(NA) {} Resources (const int& gold, const int& wood) : gold(gold), wood(wood), oil(NA) {} void clear () { gold = NA; wood = NA; oil = NA; } }; Resources resources_gathered; // Numbers currently gathered Resources resource_quotas; // Numbers required for the episode Resources requisite_resources; // Binary state variables Wargus_State(const unsigned& num_agents); unique_ptr clone () const { return unique_ptr(new Wargus_State(*this)); } unique_ptr copy () const { return unique_ptr(new Wargus_State(num_agents())); } unsigned num_agents () const { return peasants.size(); } vector variables() const; int variable_index(const string& variable_name) const; string variable_name(const int& variable_index) const; int variable_size(const int& variable_index) const; int variable(const int& variable_index) const; int& variable(const int& variable_index); map variables_mapper() const; unsigned num_agent_variables () const { return 6; } // {x, y, resource, gold_in_region, wood_in_region, townhall_in_region} set goal_variables () const { return make_set(2, variable_index("requisite_gold"), variable_index("requisite_wood")); } set transfer_variables () const { return make_set(6, variable_index("agent_resource"), variable_index("agent_region_gold"), variable_index("agent_region_wood"), variable_index("agent_region_townhall"), variable_index("requisite_gold"), variable_index("requisite_wood")); } bool action_complete(const unsigned& agent) const; Coordinate map_size() const; void draw_map() const; pair parse(string expression) const; void read(istream& in); string print() const; }; class Wargus : public MDP { protected: ClientSocket* socket; string response; int player_id; string source_map_name; unsigned num_source_maps; Wargus_State::Resources source_resource_quotas; string target_map_name; unsigned num_target_maps; Wargus_State::Resources target_resource_quotas; bool target_map_initialized; list depositing_agents; const int reward_default; const int reward_deposit; void read_socket(const bool parentheses_matching, const unsigned& num_lines = 1); int tokenizer(const string& search_str, string::size_type& index) const; void load_map(const string& map_filename); void read_map(); void state_update(); void reset(); void quit(); void kill(); void set_speed(const int& game_speed, const int& cycles_per_video_update); unsigned get_cycle(); void transition(const unsigned& num_cycles) const; void noop(const unsigned& unit_id) const; void stop(const unsigned& unit_id) const; void move(const unsigned& unit_id, const Coordinate& coord) const; bool harvest(const unsigned& unit_id, const unsigned& resource); bool deposit(const unsigned& unit_id); public: Wargus(const string& name, ClientSocket* const socket, const string& source_map_name, const unsigned& num_source_maps, const Wargus_State::Resources& source_resource_quotas, const string& target_map_name, const unsigned& num_target_maps, const Wargus_State::Resources& target_resource_quotas, const int& speed, const int& cycles); const Wargus_State& state () const { return *static_cast(_state); } // Covariant return type for easy access Wargus_State& state () { return *static_cast(_state); } // Covariant return type for easy access valarray reward() const; virtual void initialize(const bool& target = false); virtual void process(const vector& actions); bool terminated () const { return state().resources_gathered.gold >= state().resource_quotas.gold && state().resources_gathered.wood >= state().resource_quotas.wood && depositing_agents.empty(); } virtual ~Wargus (); }; hierlearning/domain/wargus/wargus_nsew.cc0000664000175000017500000000662712205065452021053 0ustar nevilleneville /************************************************* WARGUS DOMAIN WITH NSEW NAVIGATION Neville Mehta **************************************************/ #include #include "wargus_nsew.h" vector Wargus_NSEW::actions () const { vector mdp_actions; mdp_actions.push_back(North); mdp_actions.push_back(South); mdp_actions.push_back(East); mdp_actions.push_back(West); mdp_actions.push_back(MineGold); mdp_actions.push_back(ChopWood); mdp_actions.push_back(Deposit); return mdp_actions; } void Wargus_NSEW::process (const vector& actions) { for (unsigned agent = 0; agent < state().num_agents(); ++agent) { if (actions[agent] == -1) continue; if (!state().action_complete(agent)) // Ongoing primitive action { _duration[agent]++; _reward[agent] += reward_default; } else { _duration[agent] = 1.0; _reward[agent] = reward_default; switch (actions[agent]) { case North: if (state().peasants[agent].coord.y < state().map_size().y - 1) move(state().peasants[agent].id, state().peasants[agent].coord + Coordinate(0, 1)); break; case South: if (state().peasants[agent].coord.y > 0) move(state().peasants[agent].id, state().peasants[agent].coord + Coordinate(0, -1)); break; case East: if (state().peasants[agent].coord.x < state().map_size().x - 1) move(state().peasants[agent].id, state().peasants[agent].coord + Coordinate(1, 0)); break; case West: if (state().peasants[agent].coord.x > 0) move(state().peasants[agent].id, state().peasants[agent].coord + Coordinate(-1, 0)); break; case MineGold: if (state().resources_gathered.gold < state().resource_quotas.gold && state().peasants[agent].resource == Wargus_State::NL) harvest(state().peasants[agent].id, Wargus_State::Gold); break; case ChopWood: if (state().resources_gathered.wood < state().resource_quotas.wood && state().peasants[agent].resource == Wargus_State::NL) harvest(state().peasants[agent].id, Wargus_State::Wood); break; case Deposit: if (deposit(state().peasants[agent].id)) if ((state().peasants[agent].resource == Wargus_State::Gold && state().resources_gathered.gold < state().resource_quotas.gold) || (state().peasants[agent].resource == Wargus_State::Wood && state().resources_gathered.wood < state().resource_quotas.wood)) { _reward[agent] += reward_deposit; depositing_agents.push_back(agent); // Stratagus increments the gold/wood store immediately } break; } } } Wargus::process(actions); } int Wargus_NSEW::action_index (const string& action_name) const { if (action_name == "North") return North; if (action_name == "South") return South; if (action_name == "East") return East; if (action_name == "West") return West; if (action_name == "MineGold") return MineGold; if (action_name == "ChopWood") return ChopWood; if (action_name == "Deposit") return Deposit; throw HierException(__FILE__, __LINE__, "Unknown action name."); } string Wargus_NSEW::print_action (const int& action) const { switch (action) { case North: return "North"; case South: return "South"; case East: return "East"; case West: return "West"; case MineGold: return "MineGold"; case ChopWood: return "ChopWood"; case Deposit: return "Deposit"; default: throw HierException(__FILE__, __LINE__, "Unknown action."); } } hierlearning/domain/wargus/wargus_goto.h0000664000175000017500000000203312205065452020674 0ustar nevilleneville /************************************************* WARGUS DOMAIN WITH GOTO NAVIGATION Neville Mehta **************************************************/ #pragma once #include "wargus.h" class Wargus_Goto : public Wargus { public: enum Action {MineGold, ChopWood, Deposit, Move}; Wargus_Goto(const string& name, ClientSocket* const socket, const string& source_map_name, const unsigned& num_source_maps, const Wargus_State::Resources& source_resource_quotas, const string& target_map_name, const unsigned& num_target_maps, const Wargus_State::Resources& target_resource_quotas, const int& speed, const int& cycles) : Wargus(name, socket, source_map_name, num_source_maps, source_resource_quotas, target_map_name, num_target_maps, target_resource_quotas, speed, cycles) { initialize(); } vector actions() const; void initialize(const bool& target = false); void process(const vector& actions); int action_index(const string& action_name) const; string print_action(const int& action) const; ~Wargus_Goto () {} }; hierlearning/domain/wargus/module.make0000664000175000017500000000017312205065450020310 0ustar nevillenevilleFOLDER = domain/wargus OBJ = client/socket.o wargus.o wargus_goto.o wargus_nsew.o FILES += $(addprefix $(FOLDER)/, $(OBJ)) hierlearning/domain/domains.h0000664000175000017500000000030612276305042016457 0ustar nevilleneville #include "taxi/taxi.h" #include "taxi/taxi_mp.h" #include "taxi/taxi_da.h" #include "taxi/taxi_mp_da.h" #include "bitflip/bitflip.h" #include "wargus/wargus_goto.h" #include "wargus/wargus_nsew.h" hierlearning/domain/taxi/0000775000175000017500000000000012304226346015623 5ustar nevillenevillehierlearning/domain/taxi/taxi_mp_da.h0000664000175000017500000000076212205065450020103 0ustar nevilleneville /************************************************* TAXI (MOVING PASSENGER/DROPPOFF ANYWHERE) Neville Mehta Modification: Passenger moves within the taxi and can be dropped off anywhere **************************************************/ #pragma once #include "taxi.h" class Taxi_MP_DA : public Taxi { public: Taxi_MP_DA (const string& name, const double& success_probability) : Taxi(name, success_probability) {} void process(const vector& action); ~Taxi_MP_DA () {} }; hierlearning/domain/taxi/taxi.h0000664000175000017500000001020312213030340016717 0ustar nevilleneville /************************************************* TAXI DOMAIN Neville Mehta **************************************************/ /*------------------------------------------------------------------------ WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW W | W | | W 4 W R | W | | G W W_____|_____W_____|_____|_____W W | W | | W 3 W | W | | W W_____|_____W_____|_____|_____W W | | | | W 2 W |Taxi | | | W W_____|_____|_____|_____|_____W W W | W | W 1 W W | W | W W_____W_____|_____W_____|_____W W W | W | W 0 W Y W | S W B | W W W | W | W WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW 0 1 2 3 4 R, G, B, Y, (S) => Pick-up/Drop-off locations State variables: Taxi's location, fuel; Passenger's location; Passenger's destination Actions: North, South, East, West, Pick-up, Drop-off, Wait, Refuel Rewards: Successful drop-off -> Reward_Dropoff = 20 Illegal pick-up/drop-off -> Reward_Illegal = -10 Every action -> Reward_Default = -1 -------------------------------------------------------------------------*/ #pragma once #include #include #include #include #include "../../lib/common.h" #include "../../lib/matrix.h" #include "../mdp.h" struct Taxi_State : public State { int num_variables; Coordinate map_size; enum Location {NA = -2, NL, Red, Green, Blue, Yellow, Station, In_taxi}; vector site_location; int num_sites; int num_passengers; int taxi_capacity; int fuel_max; struct Taxi { Coordinate location; int fuel; } taxi; static const int num_taxi_variables = 3; struct Passenger { Coordinate location; Coordinate destination; int in_taxi; ///< Indicates passenger is in the taxi }; vector passengers; static const int num_passenger_variables = 5; static const int num_passenger_accessors = 2; Taxi_State(const Coordinate& map_size = Coordinate(5,5), const int& num_passengers = 1, const int& taxi_capacity = 1, const int& fuel_max = 0); unique_ptr clone () const { return unique_ptr(new Taxi_State(*this)); } unique_ptr copy () const { return unique_ptr(new Taxi_State(map_size, num_passengers, taxi_capacity, fuel_max)); } vector variables() const; int variable_index(const string& variable) const; string variable_name(const int& variable_index) const; int variable_size(const int& variable_index) const; int variable(const int& variable_index) const; int& variable(const int& variable_index); map variables_mapper() const; unsigned num_agent_variables () const { return num_taxi_variables; } pair parse(string expression) const; void read(istream& in); string print() const; }; /// State, reward, and time structure for the environment class Taxi : public MDP { protected: const double reward_default; ///< Default reward const double reward_dropoff; ///< For correct drop-offs const double reward_illegal; ///< For incorrect pick-ups & drop-offs const double Pr_successful_execution; ///< Probability that action is successful const bool noisy; ///< Navigation to left/right of intention matrix map_walls; void map_creator(const int& mode = 0); public: enum Action {North, South, East, West, Pickup, Dropoff, Refuel, Wait}; Taxi(const string& name, const double& success_probability); const Taxi_State& state () const { return *static_cast(_state); } // Covariant return type for easy access Taxi_State& state () { return *static_cast(_state); } // Covariant return type for easy access vector actions () const { return make_vector(6, North, South, East, West, Pickup, Dropoff); } void initialize(const bool& target = false); virtual void process(const vector& action); bool terminated() const; int action_index(const string& action_name) const; string print_action(const int& action) const; virtual ~Taxi () {} }; hierlearning/domain/taxi/taxi_da.h0000664000175000017500000000067412275613620017416 0ustar nevilleneville /************************************************* TAXI (DROP-OFF ANYWHERE) Neville Mehta Modification: Passenger can be dropped off anywhere **************************************************/ #pragma once #include "taxi.h" class Taxi_DA : public Taxi { public: Taxi_DA (const string& name, const double& success_probability) : Taxi(name, success_probability) {} void process(const vector& action); ~Taxi_DA () {} }; hierlearning/domain/taxi/taxi_mp_da.cc0000664000175000017500000000166512205065450020244 0ustar nevilleneville /************************************************* TAXI (MOVING PASSENGER/DROPPOFF ANYWHERE) Neville Mehta **************************************************/ #include "taxi_mp_da.h" void Taxi_MP_DA::process (const vector& action) { if (action[0] != Dropoff) Taxi::process(action); else // Modified dropoff { _reward = reward_default; _duration = 1.0; if (rand_real() < Pr_successful_execution) { for (auto& passenger : state().passengers) if (passenger.in_taxi) { passenger.in_taxi = false; passenger.location = state().taxi.location; // Drop off the passenger if (state().taxi.location == passenger.destination) // Taxi's at its passenger's destination _reward += reward_dropoff; else _reward += reward_illegal; } } } // Passengers move within the taxi for (auto& passenger : state().passengers) if (passenger.in_taxi) passenger.location = state().taxi.location; } hierlearning/domain/taxi/taxi_da.cc0000664000175000017500000000140512205065450017540 0ustar nevilleneville /************************************************* TAXI (DROPPOFF ANYWHERE) Neville Mehta **************************************************/ #include "taxi_da.h" void Taxi_DA::process (const vector& action) { if (action[0] != Dropoff) Taxi::process(action); else // Modified dropoff { _reward = reward_default; _duration = 1.0; if (rand_real() < Pr_successful_execution) { for (auto& passenger : state().passengers) if (passenger.in_taxi) { passenger.in_taxi = false; passenger.location = state().taxi.location; // Drop off the passenger if (state().taxi.location == passenger.destination) // Taxi's at its passenger's destination _reward += reward_dropoff; else _reward += reward_illegal; } } } } hierlearning/domain/taxi/taxi_mp.cc0000664000175000017500000000062212205065450017570 0ustar nevilleneville /************************************************* TAXI (MOVING PASSENGER) Neville Mehta **************************************************/ #include "taxi_mp.h" void Taxi_MP::process (const vector& action) { Taxi::process(action); // Passengers move within the taxi for (auto& passenger : state().passengers) if (passenger.in_taxi) passenger.location = state().taxi.location; } hierlearning/domain/taxi/taxi.cc0000664000175000017500000004570712275630276017124 0ustar nevilleneville /************************************************* TAXI DOMAIN Neville Mehta **************************************************/ #include #include "taxi.h" Taxi_State::Taxi_State (const Coordinate& map_size, const int& num_passengers, const int& taxi_capacity, const int& fuel_max) : map_size(map_size), num_passengers(num_passengers), taxi_capacity(taxi_capacity), fuel_max(fuel_max) { num_variables = num_taxi_variables + num_passengers * num_passenger_variables; num_sites = 4 + (fuel_max > 0); site_location.resize(num_sites); site_location[Red] = Coordinate(0, map_size.y - 1); site_location[Green] = Coordinate(map_size.x - 1, map_size.y - 1); site_location[Blue] = Coordinate(3 * map_size.x / 5, 0); site_location[Yellow] = Coordinate(0, 0); if (fuel_max > 0) site_location[Station] = Coordinate(2 * map_size.x / 5, 0); taxi.location = Coordinate(NA); taxi.fuel = NA; passengers.resize(num_passengers); for (auto& passenger : passengers) { passenger.location = passenger.destination = Coordinate(NA); passenger.in_taxi = NA; } } vector Taxi_State::variables () const { vector state_variables; for (int v = 0; v < num_variables; ++v) if (variable_name(v) != "agent_fuel" || fuel_max > 0) state_variables.push_back(v); return state_variables; } // Variable name to integer index int Taxi_State::variable_index (const string& variable) const { if (variable == "agent_x") return 0; if (variable == "agent_y") return 1; if (variable == "agent_fuel") return 2; if (variable.substr(0, 9) == "passenger") { int passenger_id; if (num_passengers == 1) passenger_id = 0; // In case the single passenger isn't tagged with the id else passenger_id = from_string(variable.substr(9, variable.find_first_of("_", 9) - 9)); if (passenger_id >= 0 && passenger_id < num_passengers) { if (variable.substr(variable.find_first_of("_")) == "_location_x") return num_taxi_variables + passenger_id * num_passenger_variables + 0; if (variable.substr(variable.find_first_of("_")) == "_location_y") return num_taxi_variables + passenger_id * num_passenger_variables + 1; if (variable.substr(variable.find_first_of("_")) == "_destination_x") return num_taxi_variables + passenger_id * num_passenger_variables + 2; if (variable.substr(variable.find_first_of("_")) == "_destination_y") return num_taxi_variables + passenger_id * num_passenger_variables + 3; if (variable.substr(variable.find_first_of("_")) == "_in_taxi") return num_taxi_variables + passenger_id * num_passenger_variables + 4; if (variable.substr(variable.find_first_of("_")) == "_location") return num_variables + passenger_id * num_passenger_accessors; if (variable.substr(variable.find_first_of("_")) == "_destination") return num_variables + passenger_id * num_passenger_accessors + 1; } } throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } string Taxi_State::variable_name (const int& variable_index) const { int variable = variable_index; switch (variable) { case 0: return "agent_x"; case 1: return "agent_y"; case 2: return "agent_fuel"; default: if (variable < num_variables) { variable -= num_taxi_variables; switch (variable % num_passenger_variables) { case 0: return "passenger" + (num_passengers > 1 ? to_string(variable / num_passenger_variables) : "") + "_location_x"; case 1: return "passenger" + (num_passengers > 1 ? to_string(variable / num_passenger_variables) : "") + "_location_y"; case 2: return "passenger" + (num_passengers > 1 ? to_string(variable / num_passenger_variables) : "") + "_destination_x"; case 3: return "passenger" + (num_passengers > 1 ? to_string(variable / num_passenger_variables) : "") + "_destination_y"; case 4: return "passenger" + (num_passengers > 1 ? to_string(variable / num_passenger_variables) : "") + "_in_taxi"; default: throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } else if (variable < num_variables + num_passengers * num_passenger_accessors) { variable -= num_variables; switch (variable % num_passenger_accessors) { case 0: return "passenger" + (num_passengers > 1 ? to_string(variable / num_passenger_accessors) : "") + "_location"; case 1: return "passenger" + (num_passengers > 1 ? to_string(variable / num_passenger_accessors) : "") + "_destination"; default: throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } else throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } int Taxi_State::variable_size (const int& variable_index) const { int variable = variable_index; switch (variable) { case 0: // agent_x return map_size.x; case 1: // agent_y return map_size.y; case 2: // agent_fuel return fuel_max; default: if (variable < num_variables) { variable -= num_taxi_variables; switch (variable % num_passenger_variables) { case 0: // passenger_location_x return map_size.x; case 1: // passenger_location_y return map_size.y; case 2: // passenger_destination_x return map_size.x; case 3: // passenger_destination_y return map_size.y; case 4: // passenger_in_taxi return 2; default: throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } else if (variable < num_variables + num_passengers * num_passenger_accessors) { variable -= num_variables; switch (variable % num_passenger_accessors) { case 0: // passenger_location return num_sites; case 1: // passenger_destination return num_sites; default: throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } else throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } int Taxi_State::variable (const int& variable_index) const { int variable = variable_index; switch (variable) { case 0: return taxi.location.x; case 1: return taxi.location.y; case 2: return taxi.fuel; default: if (variable < num_variables) { variable -= num_taxi_variables; switch (variable % num_passenger_variables) { case 0: // passenger_location_x return passengers[variable / num_passenger_variables].location.x; case 1: // passenger_location_y return passengers[variable / num_passenger_variables].location.y; case 2: // passenger_destination_x return passengers[variable / num_passenger_variables].destination.x; case 3: // passenger_destination_y return passengers[variable / num_passenger_variables].destination.y; case 4: // passenger_in_taxi return passengers[variable / num_passenger_variables].in_taxi; default: throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } else if (variable < num_variables + num_passengers * num_passenger_accessors) { variable -= num_variables; switch (variable % num_passenger_accessors) { case 0: // passenger_location for (int loc = 0; loc < num_sites; ++loc) if (passengers[variable / num_passenger_accessors].location == site_location[loc]) return loc; return In_taxi; case 1: // passenger_destination for (int loc = 0; loc < num_sites; ++loc) if (passengers[variable / num_passenger_accessors].destination == site_location[loc]) return loc; default: throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } else throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } int& Taxi_State::variable (const int& variable_index) { int variable = variable_index; switch (variable) { case 0: return taxi.location.x; case 1: return taxi.location.y; case 2: return taxi.fuel; default: if (variable < num_variables) { variable -= num_taxi_variables; switch (variable % num_passenger_variables) { case 0: // passenger_location_x return passengers[variable / num_passenger_variables].location.x; case 1: // passenger_location_y return passengers[variable / num_passenger_variables].location.y; case 2: // passenger_destination_x return passengers[variable / num_passenger_variables].destination.x; case 3: // passenger_destination_y return passengers[variable / num_passenger_variables].destination.y; case 4: // passenger_in_taxi return passengers[variable / num_passenger_variables].in_taxi; default: throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } else throw HierException(__FILE__, __LINE__, "Unknown variable: " + variable); } } map Taxi_State::variables_mapper () const { map var_map; for (int i = 0; i < num_variables; ++i) if (variable(i) != NA) var_map[i] = variable(i); return var_map; } pair Taxi_State::parse (string expression) const { expression = replace(expression, " ", ""); if (expression == "") throw HierException(__FILE__, __LINE__, "Expression string is empty."); int total = 0; expression.insert(0, "+"); size_t start = expression.find_first_not_of("+-", 0); size_t end = expression.find_first_of("+-", start); while (start != string::npos || end != string::npos) { istringstream token(expression.substr(start, end - start)); int value; if (!(token >> value)) // If 'token' is not an integer { if (token.str() == "NA") value = NA; else if (token.str() == "NL") value = NL; else if (token.str() == "Red") value = Red; else if (token.str() == "Green") value = Green; else if (token.str() == "Blue") value = Blue; else if (token.str() == "Yellow") value = Yellow; else if (token.str() == "Station") value = Station; else if (token.str() == "num_sites") value = num_sites; else if (token.str() == "map_size.x") value = map_size.x; else if (token.str() == "map_size.y") value = map_size.y; else return make_pair(false, 0); //throw HierException(__FILE__, __LINE__, "Cannot parse " + expression); } total += (expression[start-1] == '+') ? value : -value; start = expression.find_first_not_of("+-", end); end = expression.find_first_of("+-", start); } return make_pair(true, total); } void Taxi_State::read (istream& in) { if (in >> taxi.location) { if (fuel_max > 0) { string reader; if (in >> reader) taxi.fuel = (reader == "*") ? Taxi_State::NA : from_string(reader); else throw HierException(__FILE__, __LINE__, "Bad state read."); } for (auto& passenger : passengers) { if (in >> passenger.location) { in >> passenger.destination; in >> passenger.in_taxi; } else throw HierException(__FILE__, __LINE__, "Bad state read."); } } } string Taxi_State::print () const { ostringstream out; out << "("; if (taxi.location.x != NA) out << taxi.location.x; else out << "*"; out << ","; if (taxi.location.y != NA) out << taxi.location.y; else out << "*"; out << ")"; if (fuel_max > 0) { if (taxi.fuel != NA) out << " " << taxi.fuel; else out << " *"; } for (const auto& passenger : passengers) { out << " ("; if (passenger.location.x != NA) out << passenger.location.x; else out << "*"; out << ","; if (passenger.location.y != NA) out << passenger.location.y; else out << "*"; out << ")"; out << " ("; if (passenger.destination.x != NA) out << passenger.destination.x; else out << "*"; out << ","; if (passenger.destination.y != NA) out << passenger.destination.y; else out << "*"; out << ")"; if (passenger.in_taxi != NA) out << " " << passenger.in_taxi; else out << " *"; } return out.str(); } //************************************************************************************************* Taxi::Taxi (const string& name, const double& success_probability) : MDP(name, new Taxi_State()), reward_default(-1), reward_dropoff(20.0), reward_illegal(-10.0), Pr_successful_execution(success_probability), noisy(false) { map_creator(); initialize(); } void Taxi::map_creator (const int& mode) { map_walls.resize(2 * state().map_size.x + 1, 2 * state().map_size.y + 1); map_walls = false; for (int x = 0; x < 2 * state().map_size.x + 1; ++x) { for (int y = 0; y < 2 * state().map_size.y + 1; ++y) { // Inner walls // Original taxi world if ((y < state().map_size.y && (x == 2 * (state().map_size.x / 5) || x == 2 * (3 * state().map_size.x / 5))) || (y > state().map_size.y && x == 2 * (2 * state().map_size.x / 5))) map_walls(x,y) = true; // Corridors //if (mode == 0) //{ if (y % 2 == 0 && x != state().map_size.x) // map_walls(x,y) = true; // else // map_walls(x,y) = false; //} //else //{ if (x % 2 == 0 && y != state().map_size.y) // map_walls(x,y) = true; // else // map_walls(x,y) = false; //} // Serpentine //if (mode == 0) //{ if ((y % 4 == 0 && x > 0 && x < 2 * state().map_size.x - 1) || (y % 4 == 2 && x > 1 && x < 2 * state().map_size.x + 1)) // map_walls(x,y) = true; // else // map_walls(x,y) = false; //} //else //{ if ((x % 4 == 0 && y > 0 && y < 2 * state().map_size.y - 1) || (x % 4 == 2 && y > 1 && y < 2 * state().map_size.y + 1)) // map_walls(x,y) = true; // else // map_walls(x,y) = false; //} // Outer walls if (x == 0 || x == 2 * state().map_size.x || y == 0 || y == 2 * state().map_size.y) map_walls(x,y) = true; } } /* cout << endl; for (int y = 2 * state().map_size.y; y >= 0; --y) { for (int x = 0; x < 2 * state().map_size.x + 1; ++x) { if (x % 2 && y % 2) cout << " "; else cout << (map_walls(x,y) ? "#" : (y % 2 ? "|" : "-")) << " "; } cout << endl; } cout << endl; */ } void Taxi::initialize (const bool& target) { state().taxi.location = state().site_location[rand_int(state().num_sites)]; state().taxi.fuel = state().fuel_max > 0 ? state().fuel_max : Taxi_State::NA; for (auto& passenger : state().passengers) { int special_location = rand_int(state().num_sites); passenger.location = state().site_location[special_location]; // Passenger's destination never equals its source, except when dropped off (signaling the goal state) passenger.destination = state().site_location[(special_location + 1 + rand_int(state().num_sites - 1)) % state().num_sites]; passenger.in_taxi = false; } _reward = 0.0; _duration = 0.0; } void Taxi::process (const vector& action) { _reward = reward_default; _duration = 1.0; switch (action[0]) { case North: if (rand_real() < Pr_successful_execution) { if (!map_walls(2 * state().taxi.location.x + 1, 2 * state().taxi.location.y + 2)) ++state().taxi.location.y; } else if (noisy) { if (rand_int(2)) { if (!map_walls(2 * state().taxi.location.x, 2 * state().taxi.location.y + 1)) --state().taxi.location.x; } else if (!map_walls(2 * state().taxi.location.x + 2, 2 * state().taxi.location.y + 1)) ++state().taxi.location.x; } break; case South: if (rand_real() < Pr_successful_execution) { if (!map_walls(2 * state().taxi.location.x + 1, 2 * state().taxi.location.y)) --state().taxi.location.y; } else if (noisy) { if (rand_int(2)) { if (!map_walls(2 * state().taxi.location.x + 2, 2 * state().taxi.location.y + 1)) ++state().taxi.location.x; } else if (!map_walls(2 * state().taxi.location.x, 2 * state().taxi.location.y + 1)) --state().taxi.location.x; } break; case East: if (rand_real() < Pr_successful_execution) { if (!map_walls(2 * state().taxi.location.x + 2, 2 * state().taxi.location.y + 1)) ++state().taxi.location.x; } else if (noisy) { if (rand_int(2)) { if (!map_walls(2 * state().taxi.location.x + 1, 2 * state().taxi.location.y + 2)) ++state().taxi.location.y; } else if (!map_walls(2 * state().taxi.location.x + 1, 2 * state().taxi.location.y)) --state().taxi.location.y; } break; case West: if (rand_real() < Pr_successful_execution) { if (!map_walls(2 * state().taxi.location.x, 2 * state().taxi.location.y + 1)) --state().taxi.location.x; } else if (noisy) { if (rand_int(2)) { if (!map_walls(2 * state().taxi.location.x + 1, 2 * state().taxi.location.y)) --state().taxi.location.y; } else if (!map_walls(2 * state().taxi.location.x + 1, 2 * state().taxi.location.y + 2)) ++state().taxi.location.y; } break; case Pickup: if (rand_real() < Pr_successful_execution) { int passengers_in_taxi = 0; for (const auto& passenger : state().passengers) if (passenger.in_taxi) ++passengers_in_taxi; bool pickup_failure = true; for (int p = 0; p < state().num_passengers && passengers_in_taxi < state().taxi_capacity; ++p) if (state().taxi.location == state().passengers[p].location) // Taxi (not full) at a passenger's source location { state().passengers[p].in_taxi = true; pickup_failure = false; ++passengers_in_taxi; } if (pickup_failure) _reward += reward_illegal; } break; case Dropoff: if (rand_real() < Pr_successful_execution) { bool dropoff_failure = true; for (auto& passenger : state().passengers) if (passenger.in_taxi && state().taxi.location == passenger.destination) // Taxi's at its passenger's destination { passenger.in_taxi = false; passenger.location = state().taxi.location; // Drop off the passenger _reward += reward_dropoff; dropoff_failure = false; } if (dropoff_failure) _reward += reward_illegal; } break; case Refuel: if (rand_real() < Pr_successful_execution) if (state().taxi.location == state().site_location[Taxi_State::Station]) state().taxi.fuel = state().fuel_max; break; case Wait: break; default: throw HierException(__FILE__, __LINE__, "Unknown action."); } if (state().taxi.fuel > 0) --state().taxi.fuel; } bool Taxi::terminated () const { if (state().taxi.fuel == 0) return true; for (const auto& passenger : state().passengers) if (passenger.in_taxi || passenger.location != passenger.destination) return false; return true; } int Taxi::action_index (const string& action_name) const { if (action_name == "North") return North; if (action_name == "South") return South; if (action_name == "East") return East; if (action_name == "West") return West; if (action_name == "Pickup") return Pickup; if (action_name == "Dropoff") return Dropoff; if (action_name == "Refuel") return Refuel; if (action_name == "Wait") return Wait; throw HierException(__FILE__, __LINE__, "Unknown action name."); } string Taxi::print_action (const int& action) const { switch (action) { case North: return "North"; case South: return "South"; case East: return "East"; case West: return "West"; case Pickup: return "Pickup"; case Dropoff: return "Dropoff"; case Refuel: return "Refuel"; case Wait: return "Wait"; default: throw HierException(__FILE__, __LINE__, "Unknown action."); } } hierlearning/domain/taxi/taxi_mp.h0000664000175000017500000000066512205065450017441 0ustar nevilleneville /************************************************* TAXI (MOVING PASSENGER) Neville Mehta Modification: Passenger moves within the taxi **************************************************/ #pragma once #include "taxi.h" class Taxi_MP : public Taxi { public: Taxi_MP (const string& name, const double& success_probability) : Taxi(name, success_probability) {} void process(const vector& action); ~Taxi_MP () {} }; hierlearning/domain/taxi/module.make0000664000175000017500000000015712276304760017757 0ustar nevillenevilleFOLDER = domain/taxi OBJ = taxi.o taxi_mp.o taxi_da.o taxi_mp_da.o FILES += $(addprefix $(FOLDER)/, $(OBJ)) hierlearning/domain/mdp.h0000664000175000017500000000521312213032070015574 0ustar nevilleneville /************************************************* MARKOV DECISION PROCESS Neville Mehta **************************************************/ #pragma once #include #include #include #include #include #include using namespace std; struct State { State () {} virtual unique_ptr clone() const = 0; // Exact replicate virtual unique_ptr copy() const = 0; // Reinitialized variables virtual unsigned num_agents () const { return 1; } virtual vector variables() const = 0; virtual int variable_index(const string& variable) const = 0; virtual string variable_name(const int& variable_index) const = 0; virtual int variable_size(const int& variable_index) const = 0; virtual int variable(const int& variable_index) const = 0; virtual int& variable(const int& variable_index) = 0; virtual map variables_mapper() const = 0; virtual unsigned num_agent_variables() const = 0; virtual set goal_variables() const { return set(); } virtual set transfer_variables() const { return set(); } virtual pair parse(string expression) const = 0; virtual bool action_complete (const unsigned& agent) const { return true; } // Completion of temporally extended action in semi-MDPs virtual void read(istream& in) = 0; virtual string print() const = 0; virtual ~State () {} }; // MDP structure for the environment class MDP { protected: string _name; State* const _state; valarray _reward; valarray _duration; public: MDP (const string& name, State* const state) : _name(name), _state(state), _reward(valarray(0.0, _state->num_agents())), _duration(valarray(0.0, _state->num_agents())) {} string name () const { return _name; } virtual const State& state() const { return *_state; } virtual State& state() { return *_state; } virtual vector actions() const = 0; valarray reward () const { return _reward; } valarray duration () const { return _duration; } virtual void initialize(const bool& target = false) = 0; virtual void process(const vector& action) = 0; // Signals episode termination virtual bool terminated() const = 0; virtual int action_index(const string& action_name) const = 0; virtual string print_action(const int& action) const = 0; virtual ~MDP () { delete _state; } }; struct State_Action_Reward { unique_ptr state; vector action; valarray reward; State_Action_Reward (const State& state, const vector& action) : state(state.clone()), action(action), reward(valarray(0.0, action.size())) {} ~State_Action_Reward () {} }; hierlearning/main.cc0000664000175000017500000002155212303746004014644 0ustar nevilleneville /************************************************* MAIN Neville Mehta **************************************************/ #include #include #include #include "domain/domains.h" #include "learner/learners.h" #include "hiergen/hierarchy.h" #include "simulator.h" #if defined(NDEBUG) const bool display = false; #else const bool display = true; #endif string usage (const char* prog_name) { string program_name = to_string(prog_name); return "USAGE:\n\t" + program_name + " [OPTION] ...\n" "\t" + program_name + " -d -l [-r -e ]\n" "\t" + program_name + " -d -l [-r -s ]\n" "\t" + program_name + " -d -n -t \n" "\t" + program_name + " -d -l -n [-m ] [-r -e ]\n" "\t" + program_name + " -d -l -t [-m ] [-r -e ]\n" "\n\n" "OPTIONS:\n" "\n" "-a\n" "\tNo averaging over the runs; every run will be output in a separate file.\n" "\n" "-d \n" "\tName of the sequential decision process.\n" "\t := taxi[_mp][_stoc] | wargus_goto | bitflip\n" "\n" "\ttaxi: regular\n" "\ttaxi_mp: passenger moves within taxi\n" "\t_stoc: stochastic variant of any domain (success probability < 1 is set in the code)\n" "\tbitflip: bitflip domain\n" "\twargus_goto: resource collection in Wargus where the primitive actions are temporally-extended navigation actions\n" "\n" "-e \n" "\tNumber of simulation episodes; overrides the step switch.\n" "\n" "-h\n" "\tPrints usage information.\n" "\n" "-l \n" "\tName of the learning algorithm.\n" "\t := q | maxq | h | hh\n" "\n" "-o \n" "\tRun index; useful for execution on a cluster.\n" "\n" "-r \n" "\tNumber of simulation runs.\n" "\n" "-s \n" "\tNumber of simulation timesteps; active only when the episode switch is absent or the number of episodes is zero.\n" "\n" "-z \n" "\tRandom seed.\n" "\n\n" "HierGen:\n" "\n" "-m \n" "\tName of the directory containing previously learned models. When specified, the program avoids relearning the models from the input trajectories.\n" "\n" "-n \n" "\tNumber of trajectories to be generated. When specified (and nonzero), all switches but the trajectory filename are ignored. If the trajectory filename is specified, then the generated trajectories are saved to the file and the program terminates. If the trajectory filename is not specified, then the trajectories are generated in memory and the program continues.\n" "\n" "-t \n" "\tName of the trajectory file. If the number of trajectories is specified (and nonzero), then it indicates the output file where the generated trajectories are saved. Otherwise, it indicates the input file of trajectories.\n" "\n" "Note: If both the number of trajectories and the trajectory filename are not specified, then HierGen is not activated. The hard-coded task structure is loaded for a hierarchical agent.\n" "\n"; } int main (int argc, char* argv[]) { try { if (argc == 1) { cout << usage(argv[0]); return 1; } // Unbuffered output cout.setf(ios::unitbuf); cerr.setf(ios::unitbuf); string domain_name, learner_name, trajectory_filename, model_directory; bool averaging = true; unsigned num_runs = 0, run_index = 1, num_episodes = 0, num_steps = 0, num_trajectories = 0; // Parse command line arguments char option_char; while ((option_char = getopt(argc, argv, "ad:e:hl:m:n:o:r:s:t:z:")) != -1) { switch (option_char) { case 'a': averaging = false; break; case 'd': domain_name = optarg; break; case 'e': num_episodes = from_string(optarg); break; case 'h': cout << usage(argv[0]); return 0; case 'l': learner_name = optarg; break; case 'm': model_directory = optarg; break; case 'n': num_trajectories = from_string(optarg); break; case 'o': run_index = from_string(optarg); break; case 'r': num_runs = from_string(optarg); break; case 's': num_steps = from_string(optarg); break; case 't': trajectory_filename = optarg; break; case 'z': rand_seed(from_string(optarg)); break; case '?': throw HierException(__FILE__, __LINE__, "Unknown command-line option or argument missing."); } } unique_ptr mdp; if (domain_name.substr(0,4) == "taxi") { if (domain_name.substr(4,3) == "_mp") mdp.reset(new Taxi_MP(domain_name, (domain_name.substr(7) == "_stoc" ? 0.8 : 1.0))); else if (domain_name.substr(4,3) == "_da") mdp.reset(new Taxi_DA(domain_name, (domain_name.substr(7) == "_stoc" ? 0.8 : 1.0))); else if (domain_name.substr(4,6) == "_mp_da") mdp.reset(new Taxi_MP_DA(domain_name, (domain_name.substr(10) == "_stoc" ? 0.8 : 1.0))); else mdp.reset(new Taxi(domain_name, (domain_name.substr(4) == "_stoc" ? 0.6 : 1.0))); } else if (domain_name.substr(0,7) == "bitflip") { if (optind >= argc) throw HierException(__FILE__, __LINE__, "Insufficient arguments for bitflip: num_bits."); mdp.reset(new Bitflip(domain_name, from_string(argv[optind]))); } else if (domain_name.substr(0,6) == "wargus") { if (optind + 8 >= argc) throw HierException(__FILE__, __LINE__, "Insufficient arguments for wargus: port(default: 4870) source_map_name num_source_maps source_gold_quota source_wood_quota" "target_map_name num_target_maps target_gold_quota target_wood_quota."); if (domain_name.substr(6) == "_goto") mdp.reset(new Wargus_Goto(domain_name, new ClientSocket("localhost", from_string(argv[optind])), argv[optind + 1], from_string(argv[optind + 2]), Wargus_State::Resources(from_string(argv[optind + 3]), from_string(argv[optind + 4])), // Source argv[optind + 5], from_string(argv[optind + 6]), Wargus_State::Resources(from_string(argv[optind + 7]), from_string(argv[optind + 8])), // Target -1, display ? 1 : -1)); else if (domain_name.substr(6) == "_nsew") mdp.reset(new Wargus_NSEW(domain_name, new ClientSocket("localhost", from_string(argv[optind])), argv[optind + 1], from_string(argv[optind + 2]), Wargus_State::Resources(from_string(argv[optind + 3]), from_string(argv[optind + 4])), // Source argv[optind + 5], from_string(argv[optind + 6]), Wargus_State::Resources(from_string(argv[optind + 7]), from_string(argv[optind + 8])), // Target -1, display ? 1 : -1)); else throw HierException(__FILE__, __LINE__, "Unknown wargus MDP name."); } else throw HierException(__FILE__, __LINE__, "Incorrect program arguments. Please look at the usage information below.\n\n" + usage(argv[0])); Simulator simulator; if (num_trajectories && !trajectory_filename.empty()) simulator.trajectory_generator(trajectory_filename, *mdp, nullptr, num_trajectories); else { unique_ptr learner; // A star is born if (learner_name == "maxq" || learner_name == "hh") { unique_ptr hierarchy; if (num_trajectories && trajectory_filename.empty()) hierarchy.reset(new Hierarchy(simulator.trajectory_generator("", *mdp, nullptr, num_trajectories), model_directory, run_index, *mdp)); else if (!trajectory_filename.empty()) hierarchy.reset(new Hierarchy(trajectory_filename, model_directory, *mdp)); if (learner_name == "maxq") learner.reset(new MaxQ(learner_name, *mdp, hierarchy, run_index)); else learner.reset(new HH(learner_name, *mdp, hierarchy, run_index)); } else if (learner_name == "q" || learner_name == "h") { if (!model_directory.empty() || num_trajectories || !trajectory_filename.empty()) cerr << "Ignoring HierGen options\n"; if (learner_name == "q") learner.reset(new Q(learner_name, *mdp)); else learner.reset(new H(learner_name, *mdp)); } else throw HierException(__FILE__, __LINE__, "Incorrect program arguments. Please look at the usage information below.\n\n" + usage(argv[0])); if (num_runs && (num_episodes || num_steps)) { create_directory("results"); if (num_episodes) simulator.episode(*mdp, *learner, num_runs, run_index, num_episodes, averaging); else simulator.step(*mdp, *learner, num_runs, run_index, num_steps, averaging); learner->write_text_file("results/learner_" + learner->name() + "_" + mdp->name() + ".out", *mdp); } } } catch (const exception& e) { cerr << e.what() << endl << endl; } catch (...) { cerr << "\n Unhandled exception.\n"; } return 0; } hierlearning/model_builder.py0000664000175000017500000001370412303746004016571 0ustar nevilleneville # Model Builder # Converts trajectories to arff files --- one for each action-variable pair --- and builds J48 decision-tree action models # Compatible with Weka 3.6.5 # Note: numeric attributes are not allowed for J48 import sys import subprocess import re import os import platform oper_sys = platform.system() if len(sys.argv) != 5: print "Arguments: " sys.exit(1) confidence = float(sys.argv[3]) prune = confidence != 0.0 display = sys.argv[4] == "True" if not os.access(sys.argv[1], os.R_OK): print "Trajectory file cannot be read." sys.exit(1) if not os.access(sys.argv[2], os.W_OK): os.mkdir(sys.argv[2]) infile = open(sys.argv[1], 'r') data = infile.readlines(); infile.close() pattern = re.compile(r'[ ]+') actions = pattern.split(data[0].strip()) num_actions = len(actions) action_encoding = dict(zip(map(int, pattern.split(data[1].strip())), [i for i in range(num_actions)])) state_variables = pattern.split(data[2].strip()) num_variables = len(state_variables) domain_size = map(int, pattern.split(data[3].strip())) if num_variables != len(domain_size): print "The number of variables does not match the number of domain sizes." sys.exit(1) directory = sys.argv[2] + '/' if display: sys.stdout.write("Creating arff files ... ") arff_file = [] # Transition data for a in range(num_actions): arff_file.append([]) for x in range(num_variables): arff_file[a].append(open(directory + actions[a] + '__' + state_variables[x] + '.arff', 'w')) arff_file[a][x].write('@relation ' + actions[a] + '__' + state_variables[x] + '\n') for v in range(num_variables): arff_file[a][x].write('@attribute ' + state_variables[v] + ' {' + str(range(domain_size[v])).strip('[]') + '}\n') arff_file[a][x].write('@attribute next_' + state_variables[x] + ' {' + str(range(domain_size[x])).strip('[]') + '}\n\n@data\n') reward_nominal_values = [] reward_data = [] for a in range(num_actions): reward_nominal_values.append(set()) reward_data.append("") pattern = re.compile(r'[ \(\),\*]+') for i in range(5, len(data) - 3): state = [s for s in pattern.split(data[i].strip()) if s] if len(state) > 1: if len(state) != num_variables: print "The state does not contain the expected number of state variables." sys.exit(1) if data[i+1].strip() == '#': continue a = action_encoding[int(data[i+1].strip())] reward = float(data[i+2].strip()) next_state = [s for s in pattern.split(data[i+3].strip()) if s] # Transition arff for x in range(num_variables): for s in state: arff_file[a][x].write(s + ',') arff_file[a][x].write(next_state[x] + '\n') # Reward arff for s in state: reward_data[a] += s + ',' reward_data[a] += str(reward) + '\n' reward_nominal_values[a].add(reward) for a in range(num_actions): for x in range(num_variables): arff_file[a][x].close() # Reward data for a in range(num_actions): reward_arff_file = open(directory + actions[a] + '__reward.arff', 'w') reward_arff_file.write('@relation ' + actions[a] + '__reward\n') for v in range(num_variables): reward_arff_file.write('@attribute ' + state_variables[v] + ' {' + str(range(domain_size[v])).strip('[]') + '}\n') reward_arff_file.write('@attribute reward {' + str(reward_nominal_values[a]).strip('set()[]') + '}\n\n@data\n') reward_arff_file.write(reward_data[a]) reward_arff_file.close() if display: print "done." # Process arff files if display: print "Creating models ..." line_pattern = re.compile(r'[\r\n]*') count_pattern = re.compile(r' \(.*\)') for a in range(num_actions): if display: print actions[a] for x in range(num_variables): if not prune: model_str = subprocess.Popen(["java", "weka.classifiers.trees.J48", "-v", "-no-cv", "-U", "-M", "1", "-t", directory + actions[a] + "__" + state_variables[x] + ".arff"], stdout = subprocess.PIPE).communicate()[0] else: model_str = subprocess.Popen(["java", "weka.classifiers.trees.J48", "-v", "-no-cv", "-C", str(confidence), "-M", "1", "-t", directory + actions[a] + "__" + state_variables[x] + ".arff"], stdout = subprocess.PIPE).communicate()[0] if model_str != "": model = line_pattern.split(model_str) model_file = open(directory + actions[a] + '__' + state_variables[x] + '.model', 'w') for line in model[4:len(model) - 3]: if line.find(' (0.0)') == -1: model_file.write(count_pattern.split(line)[0] + '\n') model_file.close() if len(reward_nominal_values[a]) > 1: if not prune: model_str = subprocess.Popen(["java", "weka.classifiers.trees.J48", "-v", "-no-cv", "-U", "-M", "1", "-t", directory + actions[a] + "__reward.arff"], stdout = subprocess.PIPE).communicate()[0] else: model_str = subprocess.Popen(["java", "weka.classifiers.trees.J48", "-v", "-no-cv", "-C", str(confidence), "-M", "1", "-t", directory + actions[a] + "__reward.arff"], stdout = subprocess.PIPE).communicate()[0] if model_str != "": model = line_pattern.split(model_str) model_file = open(directory + actions[a] + '__reward.model', 'w') for line in model[4:len(model) - 3]: if line.find(' (0.0)') == -1: model_file.write(count_pattern.split(line)[0] + '\n') model_file.close() else: model_file = open(directory + actions[a] + '__reward.model', 'w') model_file.write(': ' + str(reward_nominal_values[a]).strip('set()[]') + '\n') model_file.close() if display: print "Done." hierlearning/hierlearning.vcxproj0000664000175000017500000002255212303746004017476 0ustar nevilleneville Debug Win32 Release Win32 {948C4148-9B3E-47D5-A5DC-373FD22F71AF} hierlearning Win32Proj hierlearning Application Unicode true v110 Application Unicode v110 <_ProjectFileVersion>10.0.30319.1 $(SolutionDir)$(Configuration)\ $(Configuration)\ true $(SolutionDir)$(Configuration)\ $(Configuration)\ false Disabled %(AdditionalIncludeDirectories) WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true EnableFastChecks MultiThreadedDebugDLL Level3 4250 ws2_32.lib;%(AdditionalDependencies) true Console false MachineX86 WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) MultiThreadedDLL Level3 ProgramDatabase true Console true true false MachineX86 ws2_32.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) hierlearning/lib/0000775000175000017500000000000012304226346014155 5ustar nevillenevillehierlearning/lib/common.cc0000664000175000017500000001133212205331206015744 0ustar nevilleneville /*************************************** COMMON FUNCTIONS Neville Mehta ****************************************/ #include #include #include #include "common.h" namespace { random_device rand_dev; mt19937 rand_engine(rand_dev()); uniform_real_distribution ur_dist; normal_distribution n_dist; } void rand_seed (const unsigned& index) { rand_engine.seed(index + 1); } unsigned rand_int (const unsigned& range) // Integer random value {0, ..., range - 1} { uniform_int_distribution ui_dist(0, range - 1); return ui_dist(rand_engine); } double rand_real () // Real random value [0, 1) { return ur_dist(rand_engine); } double rand_norm () // Standard Normal random value { return n_dist(rand_engine); } #if !defined(__unix__) char* optarg; int optind = 0; static char* scan = 0; static char* index (const char* str, int chr) { while (*str) { if (*str == chr) return (char*)str; ++str; } return 0; } int getopt (int argc, char* argv[], char* optstring) { register char c; register char* place; optarg = 0; if (scan == 0 || *scan == '\0') { if (optind == 0) ++optind; if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') return -1; if (strcmp(argv[optind], "--") == 0) { ++optind; return -1; } scan = argv[optind] + 1; ++optind; } c = *scan++; place = index(optstring, c); if (place == 0 || c == ':') return '?'; ++place; if (*place == ':') { if (*scan != '\0') { optarg = scan; scan = 0; } else if (optind < argc) { optarg = argv[optind]; ++optind; } else return '?'; } return c; } #endif istream& operator >> (istream& in, Coordinate& coord) { string reader; if (in >> reader) { string::size_type start = reader.find_first_not_of("("); string::size_type end = reader.find_first_of(",", start); coord.x = (reader.substr(start, end - start) == "*") ? -1 : from_string(reader.substr(start, end - start)); start = end + 1; end = reader.find_first_of(")", start); coord.y = (reader.substr(start, end - start) == "*") ? -1 : from_string(reader.substr(start, end - start)); } return in; } ostream& operator << (ostream& out, const Coordinate& coord) { out << "(" << coord.x << "," << coord.y << ")"; return out; } void getline_os (istream& in, string& str) { if (in.bad()) throw HierException(__FILE__, __LINE__, "Bad istream."); getline(in, str); if (!str.empty() && str[str.size() - 1] == '\r') str.resize(str.size() - 1); } string replace (const string& str, const string& from, const string& to) { string new_str(str); string::size_type pos = 0; while ((pos = new_str.find(from, pos)) != string::npos) { new_str.replace(pos, from.length(), to); pos += from.length(); } return new_str; } vector tokenize (const string& str, const string& token_separators) { vector tokens; string::size_type start = str.find_first_not_of(token_separators, 0); string::size_type end = str.find_first_of(token_separators, start); while (start != string::npos) { tokens.push_back(str.substr(start, end - start)); start = str.find_first_not_of(token_separators, end); end = str.find_first_of(token_separators, start); } return tokens; } bool entity_exists (const string& name) { struct stat entity_info; if (stat(name.c_str(), &entity_info)) return false; return true; } void create_directory (const string& directory) { if (!directory.empty() && !entity_exists(directory)) { #if defined __unix__ if (system(("mkdir -p " + directory).c_str())) #else string directory_windows = replace(directory, "/", "\\"); if (system(("mkdir " + directory_windows).c_str())) #endif throw HierException(__FILE__, __LINE__, "Unable to create the " + directory + " directory"); } } void remove_directory (const string& directory) { if (entity_exists(directory)) { #if defined __unix__ if (system(("rm -r " + directory).c_str())) #else string directory_windows = replace(directory, "/", "\\"); if (system(("rmdir /s /q " + directory_windows).c_str())) #endif cerr << "(" + to_string(__FILE__) + ":" + to_string(__LINE__) + ")\n"; } } void file_table (const valarray& table, const string& filename) { ofstream output(filename.c_str()); if (!output.is_open()) throw HierException(__FILE__, __LINE__, "Unable to write to " + filename); for (unsigned i = 0; i < table.size(); ++i) output << i << "\t" << table[i] << endl; output.close(); } bool is_equal_double (const double& a, const double& b) { // Best way to compare floating point numbers if (fabs(a - b) < 1e-8 // absolute error && ((b > 0 && a > b * (1 - 1e10) && a < b * (1 + 1e10)) || (b < 0 && a > b * (1 + 1e10) && a < b * (1 - 1e10)))) // relative error return true; return false; } hierlearning/lib/matrix.h0000664000175000017500000002001012205065452015622 0ustar nevilleneville /*************************************** MATRIX Neville Mehta ****************************************/ #pragma once #include #include #include #include "common.h" class MatrixException : public exception { string error; public: MatrixException (const string& error = "") : error("\nMatrix Exception: " + error) {} const char* what () const throw() { return error.c_str(); } ~MatrixException () throw() {} }; template class matrix { protected: unsigned _num_rows, _num_cols; valarray* _data; public: explicit matrix (const unsigned& num_rows = 1, const unsigned& num_cols = 1) : _num_rows(num_rows), _num_cols(num_cols), _data(new valarray(T(), num_rows * num_cols)) {} matrix (const unsigned& num_rows, const unsigned& num_cols, const T& val) : _num_rows(num_rows), _num_cols(num_cols), _data(new valarray(val, _num_rows * _num_cols)) {} matrix (const unsigned& num_rows, const unsigned& num_cols, const T* v) : _num_rows(num_rows), _num_cols(num_cols), _data(new valarray(v, _num_rows * _num_cols)) {} matrix (const unsigned& num_rows, const unsigned& num_cols, const valarray& v) : _num_rows(num_rows), _num_cols(num_cols) { if (v.size() != _num_rows * _num_cols) throw MatrixException("Illegal matrix construction."); else _data = new valarray(v); } matrix (const matrix& m) : _num_rows(m._num_rows), _num_cols(m._num_cols), _data(new valarray(*m._data)) {} const valarray data () const { return *_data; } unsigned num_rows () const { return _num_rows; } unsigned num_cols () const { return _num_cols; } unsigned size () const { return _num_rows * _num_cols; } void resize (const unsigned& num_rows, const unsigned& num_cols) { valarray temp(*_data); // Saves the old data before the resizing operation wipes it completely _data->resize(num_rows * num_cols); // Direct assignment of differing-size valarrays has undefined behavior unsigned min_size = _data->size() < temp.size() ? _data->size() : temp.size(); for (unsigned i = 0; i < min_size; ++i) (*_data)[i] = temp[i]; _num_rows = num_rows; _num_cols = num_cols; } void remove_row (const unsigned& row_index) { for (unsigned r = row_index + 1; r < _num_rows; ++r) for (unsigned c = 0; c < _num_cols; ++c) (*_data)[(r - 1) * _num_cols + c] = (*_data)[r * _num_cols + c]; resize(_num_rows - 1, _num_cols); } void remove_column (const unsigned& column_index) { for (unsigned r = 0; r < _num_rows; ++r) for (unsigned c = 0; c < _num_cols; ++c) if (c != column_index) (*_data)[r * (_num_cols - 1) + (c > column_index ? c - 1 : c)] = (*_data)[r * _num_cols + c]; resize(_num_rows, _num_cols - 1); } T operator () (const unsigned& r, const unsigned& c) const { if (r >= _num_rows || c >= _num_cols) throw MatrixException("Matrix indices out of bounds."); return (*_data)[r * _num_cols + c]; } T& operator () (const unsigned& r, const unsigned& c) { if (r >= _num_rows || c >= _num_cols) throw MatrixException("Matrix indices out of bounds."); return (*_data)[r * _num_cols + c]; } const valarray operator [] (const unsigned& row) const { return (*_data)[slice(row * _num_cols, _num_cols, 1)]; } void set_row (const unsigned& row, const valarray& v) { if (_num_cols != v.size()) throw MatrixException("Dimensions not matched for row assignment."); (*_data)[slice(row * _num_cols, _num_cols, 1)] = v; } const valarray col (const unsigned& column) const { return (*_data)[slice(column, _num_rows, _num_cols)]; } void set_col (const unsigned& column, const valarray& v) { if (_num_rows != v.size()) throw MatrixException("Dimensions not matched for column assignment."); return (*_data)[slice(column, _num_rows, _num_cols)] = v; } // Scalar assignment matrix& operator = (const T& val) { *_data = val; return *this; } // Matrix assignment matrix& operator = (const matrix& m) { if (this != &m) { if (_num_rows != m._num_rows || _num_cols != m._num_cols) throw MatrixException("Dimensions not matched for assignment."); *_data = *m._data; } return *this; } // Equals bool operator == (const matrix& m) const { if (_num_rows == m._num_rows && _num_cols == m._num_cols) return (*_data == *m._data).min(); return false; } bool operator != (const matrix& m) const { return !(*this == m); } // Transpose matrix operator ~ () const { matrix result(_num_cols, _num_rows); for (unsigned i = 0; i < _num_rows; ++i) result.set_col(i, (*this)[i]); return result; } T sum () const { return _data->sum(); } T min () const { return _data->min(); } T max () const { return _data->max(); } // Addition matrix operator + (const matrix& m) const { if (_num_rows != m._num_rows || _num_cols != m._num_cols) throw MatrixException("Dimensions not matched for addition."); return matrix(_num_rows, _num_cols, (*_data) + *m._data); } // Add & assign matrix& operator += (const matrix& m) { if (_num_rows != m._num_rows || _num_cols != m._num_cols) throw MatrixException("Dimensions not matched for addition."); *_data += *m._data; return *this; } // Subtraction matrix operator - (const matrix& m) const { if (_num_rows != m.num_rows() || _num_cols != m.num_cols()) throw MatrixException("Dimensions not matched for subtraction."); return matrix(_num_rows, _num_cols, (*_data) - m.data()); } // Subtract & assign matrix& operator -= (const matrix& m) { if (_num_rows != m._num_rows || _num_cols != m._num_cols) throw MatrixException("Dimensions not matched for subtraction."); *_data -= *m._data; return *this; } // Multiplication by scalar matrix operator * (const T& val) const { return matrix(_num_rows, _num_cols, (*_data) * val); } // Multiplication by scalar (*=) matrix& operator *= (const T& val) { *_data *= val; return *this; } // Multiplication by vector matrix operator * (const valarray& v) const { if (_num_cols != v.size()) throw MatrixException("Dimensions not matched for multiplication."); matrix result(_num_rows, 1); for (unsigned i = 0; i < _num_rows; ++i) result(i, 0) = ((*this)[i] * v).sum(); return result; } // Multiplication by matrix matrix operator * (const matrix& m) const { if (_num_cols != m._num_rows) throw MatrixException("Dimensions not matched for multiplication."); matrix result(_num_rows, m._num_cols); for (unsigned i = 0; i < _num_rows; ++i) for (unsigned j = 0; j < m._num_cols; ++j) result(i, j) = ((*this)[i] * m.col(j)).sum(); return result; } // Division by scalar matrix operator / (const T& val) const { return matrix(_num_rows, _num_cols, (*_data) / val); } // Division by scalar (/=) matrix& operator /= (const T& val) { *_data /= val; return *this; } // Component-wise division by vector matrix operator / (const valarray& v) const { if (_num_rows != v.size()) throw MatrixException("Dimensions not matched for component-wise division."); matrix result(_num_rows, _num_cols); for (unsigned i = 0; i < _num_rows; ++i) result.set_row(i, (*this)[i] / v[i]); return result; } ~matrix () { delete _data; } }; // Absolute template inline matrix abs (const matrix& m) { return matrix(m.num_rows(), m.num_cols(), abs(m.data())); } // Multiplication by scalar template inline matrix operator * (const T& val, const matrix& m) { return m * val; } template istream& operator >> (istream& istr, matrix& m) { for (unsigned i = 0; i < m.num_rows(); ++i) for (unsigned j = 0; j < m.num_cols(); ++j) istr >> m(i, j); return istr; } template ostream& operator << (ostream& out, const matrix& m) { for (unsigned i = 0; i < m.num_rows(); ++i) { out << "\n["; for (unsigned j = 0; j < m.num_cols(); ++j) out << ' ' << m(i, j); out << " ]"; } return out; } hierlearning/lib/graph.h0000664000175000017500000002630012205065452015427 0ustar nevilleneville /*************************************** GRAPH Neville Mehta ****************************************/ #pragma once #include #include #include #include #include #include #include "matrix.h" class GraphException : public exception { private: string error; public: GraphException (const string& error = "") : error("Graph Exception: " + error) {} const char* what () const throw() { return error.c_str(); } ~GraphException () throw() {} }; template class graph { matrix _adj_mat; // Adjacency matrix set> tarjan (const unsigned& i, int& index, vector& v_low, vector& v_index, list& v_stack) const { set> scc; v_low[i] = index; v_index[i] = index; ++index; v_stack.push_back(i); set child_set = children(i); for (auto n_itr = child_set.cbegin(); n_itr != child_set.cend(); ++n_itr) { if (v_index[*n_itr] == -1) { set> sub_scc = tarjan(*n_itr, index, v_low, v_index, v_stack); scc.insert(sub_scc.begin(), sub_scc.end()); v_low[i] = min(v_low[i], v_low[*n_itr]); } else if (find(v_stack.begin(), v_stack.end(), *n_itr) != v_stack.end()) v_low[i] = min(v_low[i], v_index[*n_itr]); } if (v_low[i] == v_index[i]) { set component; unsigned j; do { j = v_stack.back(); component.insert(j); v_stack.pop_back(); } while (j != i); scc.insert(component); } return scc; } // Composition of graphs (relations) g2 o g1: if there exists y such that (x,y) in g1 and (y,z) in g2, then (x,z) is in the result graph composition (const graph& g, const T& edge_label) const { graph gc(num_vertices()); for (unsigned i = 0; i < num_vertices(); ++i) for (unsigned j = 0; j < num_vertices(); ++j) for (unsigned k = 0; k < num_vertices(); ++k) if (g.edge_exists(i, k, edge_label) && edge_exists(k, j, edge_label)) { gc.set_edge(i, j, edge_label); break; } return gc; } public: explicit graph (const unsigned& num_vertices = 1) : _adj_mat(matrix(num_vertices, num_vertices, static_cast(nullptr))) {} graph (const graph& g) : _adj_mat(matrix(g.num_vertices(), g.num_vertices(), static_cast(nullptr))) { for (unsigned i = 0; i < num_vertices(); ++i) for (unsigned j = 0; j < num_vertices(); ++j) if (g._adj_mat(i,j)) _adj_mat(i,j) = new T(*g._adj_mat(i,j)); } graph& operator = (const graph& g) { if (this != &g) { for (unsigned i = 0; i < num_vertices(); ++i) for (unsigned j = 0; j < num_vertices(); ++j) if (_adj_mat(i, j)) { delete _adj_mat(i, j); _adj_mat(i, j) = nullptr; } _adj_mat.resize(g.num_vertices(), g.num_vertices()); for (unsigned i = 0; i < num_vertices(); ++i) for (unsigned j = 0; j < num_vertices(); ++j) if (g._adj_mat(i,j)) _adj_mat(i,j) = new T(*g._adj_mat(i,j)); } return *this; } unsigned num_vertices () const { return _adj_mat.num_rows(); } void resize (const unsigned& num_vertices) { _adj_mat.resize(num_vertices, num_vertices); } void set_edge (const unsigned& v1, const unsigned& v2, const T& edge_label = T()) { if (v1 >= num_vertices() || v2 >= num_vertices()) throw GraphException("Vertex indices out of bounds."); if (!_adj_mat(v1, v2)) _adj_mat(v1, v2) = new T; *_adj_mat(v1, v2) = edge_label; } bool edge_exists (const unsigned& v1, const unsigned& v2) const { if (v1 >= num_vertices() || v2 >= num_vertices()) throw GraphException("Vertex indices out of bounds."); return _adj_mat(v1, v2) != nullptr; } bool edge_exists (const unsigned& v1, const unsigned& v2, const T& edge_label) const { if (v1 >= num_vertices() || v2 >= num_vertices()) throw GraphException("Vertex indices out of bounds."); return _adj_mat(v1, v2) && *_adj_mat(v1, v2) == edge_label; } T& edge (const unsigned& v1, const unsigned& v2) const { if (v1 >= num_vertices() || v2 >= num_vertices()) throw GraphException("Vertex indices out of bounds."); if (!_adj_mat(v1, v2)) throw GraphException("Edge does not exist."); return *_adj_mat(v1, v2); } void remove_edge (const unsigned& v1, const unsigned& v2) { if (v1 >= num_vertices() || v2 >= num_vertices()) throw GraphException("Vertex indices out of bounds."); if (_adj_mat(v1, v2)) { delete _adj_mat(v1, v2); _adj_mat(v1, v2) = nullptr; } } void remove_edge (const unsigned& v1, const unsigned& v2, const T& edge_label) { if (v1 >= num_vertices() || v2 >= num_vertices()) throw GraphException("Vertex indices out of bounds."); if (edge_exists(v1, v2, edge_label)) { delete _adj_mat(v1, v2); _adj_mat(v1, v2) = nullptr; } } void remove_node (const unsigned& v) { // Remove all arcs associate with v for (unsigned i = 0; i < num_vertices(); ++i) { if (_adj_mat(v, i)) { delete _adj_mat(v, i); _adj_mat(v, i) = nullptr; } if (_adj_mat(i, v)) { delete _adj_mat(i, v); _adj_mat(i, v) = nullptr; } } _adj_mat.remove_row(v); _adj_mat.remove_column(v); } set children (const unsigned& v) const { set neighbors; for (unsigned w = 0; w < num_vertices(); ++w) if (_adj_mat(v, w)) neighbors.insert(w); return neighbors; } set children (const unsigned& v, const T& edge_label) const { set neighbors; for (unsigned w = 0; w < num_vertices(); ++w) if (edge_exists(v, w, edge_label)) neighbors.insert(w); return neighbors; } set parents (const unsigned& v) const { set neighbors; for (unsigned w = 0; w < num_vertices(); ++w) if (_adj_mat(w, v)) neighbors.insert(w); return neighbors; } set parents (const unsigned& v, const T& edge_label) const { set neighbors; for (unsigned w = 0; w < num_vertices(); ++w) if (edge_exists(w, v, edge_label)) neighbors.insert(w); return neighbors; } // Returns empty vector if no path exists vector shortest_path (const unsigned& start_vertex, const unsigned& end_vertex) const { if (start_vertex == end_vertex) return vector(1, start_vertex); vector parent(num_vertices(), -1); parent[start_vertex] = start_vertex; queue next_vertex; next_vertex.push(start_vertex); while (!next_vertex.empty() && next_vertex.front() != end_vertex) { unsigned i = next_vertex.front(); next_vertex.pop(); for (unsigned j = 0; j < num_vertices(); ++j) if (edge_exists(i, j) && parent[j] == -1) { parent[j] = i; next_vertex.push(j); } } vector sh_path; if (!next_vertex.empty() && next_vertex.front() == end_vertex) // Path exists { sh_path.push_back(end_vertex); unsigned i = end_vertex; while (i != start_vertex) { sh_path.insert(sh_path.begin(), parent[i]); i = parent[i]; } } return sh_path; } // Returns empty vector if no path exists vector shortest_path (const unsigned& start_vertex, const unsigned& end_vertex, const T& edge_label) const { if (start_vertex == end_vertex) return vector(1, start_vertex); vector parent(num_vertices(), -1); parent[start_vertex] = start_vertex; queue next_vertex; next_vertex.push(start_vertex); while (!next_vertex.empty() && next_vertex.front() != end_vertex) { unsigned i = next_vertex.front(); next_vertex.pop(); for (unsigned j = 0; j < num_vertices(); ++j) if (edge_exists(i, j, edge_label) && parent[j] == -1) { parent[j] = i; next_vertex.push(j); } } vector sh_path; if (!next_vertex.empty() && next_vertex.front() == end_vertex) // Path exists { sh_path.push_back(end_vertex); unsigned i = end_vertex; while (i != start_vertex) { sh_path.insert(sh_path.begin(), parent[i]); i = parent[i]; } } return sh_path; } // Set of vertex sets of strongly connected components set> strongly_connected_components () const { set> all_sccs; vector v_low(num_vertices(), 0); vector v_index(num_vertices(), -1); for (unsigned v = 0; v < num_vertices(); ++v) if (v_index[v] == -1) { int index = 0; list v_stack; set> scc = tarjan(v, index, v_low, v_index, v_stack); all_sccs.insert(scc.begin(), scc.end()); } return all_sccs; } bool is_strongly_connected () const { vector v_low(num_vertices(), 0); vector v_index(num_vertices(), -1); list v_stack; int index = 0; set> scc = tarjan(0, index, v_low, v_index, v_stack); // Only need to check one vertex if (scc.begin()->size() == num_vertices()) return true; return false; } bool is_acyclic () const { for (unsigned i = 0; i < num_vertices(); ++i) if (edge_exists(i, i)) return false; return strongly_connected_components().size() == num_vertices(); } // Transitive closure of edges with the specified label (Warshall's algorithm in place) void transitive_closure (const T& edge_label) { for (unsigned j = 0; j < num_vertices(); ++j) for (unsigned i = 0; i < num_vertices(); ++i) if (i != j && edge_exists(i, j, edge_label)) for (unsigned k = 0; k < num_vertices(); ++k) if (k != j && !edge_exists(i, k, edge_label) && edge_exists(j, k, edge_label)) set_edge(i, k, edge_label); } // Transitive reduction = g - (g o g*) where g* is the transitive closure void transitive_reduction (const T& edge_label) { graph g(*this); g.transitive_closure(edge_label); g = composition(g, edge_label); for (unsigned i = 0; i < num_vertices(); ++i) for (unsigned j = 0; j < num_vertices(); ++j) if (g.edge_exists(i, j, edge_label)) remove_edge(i, j, edge_label); } void print_gv (const string& filename, const bool& edge_label = false) const { ofstream gfile(filename.c_str()); if (!gfile.is_open()) throw GraphException("Unable to write to file."); gfile << "digraph {\nedge [arrowhead=empty]\n\n"; for (unsigned i = 0; i < num_vertices(); ++i) for (unsigned j = 0; j < num_vertices(); ++j) if (_adj_mat(i, j)) { gfile << i << " -> " << j; if (edge_label) gfile << " [label=\"" << *_adj_mat(i,j) << "\"]\n"; else gfile << endl; } gfile << "}\n"; gfile.close(); } ~graph () { for (unsigned i = 0; i < num_vertices(); ++i) for (unsigned j = 0; j < num_vertices(); ++j) if (_adj_mat(i, j)) delete _adj_mat(i, j); } }; template istream& operator >> (istream& istr, graph& g) { T edge_label; for (unsigned i = 0; i < g.num_vertices(); ++i) for (unsigned j = 0; j < g.num_vertices(); ++j) { istr >> edge_label; g.set_edge(i, j, edge_label); } return istr; } template ostream& operator << (ostream& out, const graph& g) { out << "Vertices = {"; for (unsigned i = 0; i < g.num_vertices(); ++i) { if (i > 0) out << ", "; out << i; } out << "}\nEdges = {"; for (unsigned i = 0; i < g.num_vertices(); ++i) { for (unsigned j = 0; j < g.num_vertices(); ++j) if (g.edge_exists(i, j)) out << "\n" << i << " -> " << j << ":\n" << g.edge(i, j); } out << "\n}\n"; return out; } hierlearning/lib/common.h0000664000175000017500000001152212205331722015612 0ustar nevilleneville /*************************************** COMMON FUNCTIONS Neville Mehta ****************************************/ #pragma once #include #include #include #include #include #include #include #include #include #include using namespace std; void rand_seed(const unsigned& index); unsigned rand_int(const unsigned& range); double rand_real(); double rand_norm(const double& mean, const double& variance); void getline_os(istream& in, string& str); string replace(const string& str, const string& from, const string& to); vector tokenize(const string& str, const string& token_separators); bool entity_exists(const string& name); // File or directory void create_directory(const string& directory); void remove_directory(const string& directory); void file_table(const valarray& table, const string& fname); bool is_equal_double(const double& a, const double& b); #if defined(__unix__) #include #else extern int optind; extern char* optarg; int getopt(int argc, char* argv[], char* optstring); #endif class HierException : public exception { string filename; int line; string error; public: HierException (const string& filename, const int& line, const string& error) : filename(filename), line(line), error(error) {} const char* what () const throw() { static string output; output = "\n" + error + " (" + filename + ":" + to_string(line) + ")"; return output.c_str(); } virtual ~HierException () throw() {} }; struct Coordinate { int x; int y; Coordinate (const int& val = 0) : x(val), y(val) {} Coordinate (const int& x, const int& y) : x(x), y(y) {} bool operator == (const Coordinate& coord) const { return x == coord.x && y == coord.y; } bool operator != (const Coordinate& coord) const { return x != coord.x || y != coord.y; } // Alternatively: return !(*this == coord); Coordinate operator + (const Coordinate& coord) const { return Coordinate(x + coord.x, y + coord.y); } Coordinate& operator += (const Coordinate& coord) { x += coord.x; y += coord.y; return *this; } template Coordinate operator / (const T& val) const { return Coordinate(x / val, y / val); } template Coordinate operator % (const T& val) const { return Coordinate(x % val, y % val); } }; istream& operator >>(istream& in, Coordinate& coord); ostream& operator <<(ostream& out, const Coordinate& coord); template string to_string (const T& thing) { ostringstream out_stream; out_stream << thing; return out_stream.str(); } template T from_string (const string& str) { istringstream in_stream(str); T value; if (in_stream >> value) return value; throw HierException(__FILE__, __LINE__, "Type " + to_string(typeid(T).name()) + " expected in the string \"" + str + "\"."); } inline int combination (const int& n, const int& r) { int numerator = 1, denominator = 1; if (r < n / 2) { for (int x = n - r + 1; x <= n; ++x) numerator *= x; for (int x = 2; x <= r; ++x) denominator *= x; } else { for (int x = r + 1; x <= n; ++x) numerator *= x; for (int x = 2; x <= n - r; ++x) denominator *= x; } return numerator / denominator; } template set make_set (unsigned num_args, ...) { va_list vl; va_start(vl, num_args); set s; while (num_args > 0) { s.insert(va_arg(vl, T)); --num_args; } va_end(vl); return s; } template vector make_vector (unsigned num_args, ...) { va_list vl; va_start(vl, num_args); vector v; while (num_args > 0) { v.push_back(va_arg(vl, T)); --num_args; } va_end(vl); return v; } template set extract_map_keys (const map& m) { set keys; for (const auto& x : m) keys.insert(x.first); return keys; } template set extract_map_values (const map& m) { set values; for (const auto& x : m) values.insert(x.second); return values; } template ostream& operator << (ostream& out, const valarray& v) { out << "("; for (size_t x = 0; x < v.size(); ++x) { if (x > 0) out << ", "; out << v[x]; } out << ")"; return out; } template ostream& operator << (ostream& out, const vector& v) { out << "("; for (auto x = v.cbegin(); x != v.cend(); ++x) { if (x != v.begin()) out << ","; out << *x; } out << ")"; return out; } template ostream& operator << (ostream& out, const map& m) { out << "("; for (auto x = m.cbegin(); x != m.cend(); ++x) { if (x != m.begin()) out << ","; out << "(" << x->first << ":" << x->second << ")"; } out << ")"; return out; } template ostream& operator << (ostream& out, const set& s) { out << "{"; for (auto x = s.cbegin(); x != s.cend(); ++x) { if (x != s.begin()) out << ","; out << *x; } out << "}"; return out; } hierlearning/dotify_batch0000664000175000017500000000012312303746004015762 0ustar nevilleneville#!/bin/sh ls $1/*.gv | sed -e "s/.gv//" | xargs -Ixxx dot -Tpdf -o xxx.pdf xxx.gv hierlearning/README.txt0000664000175000017500000000434412303746004015107 0ustar nevillenevilleHierLearning is a C++11 implementation of a general-purpose, multi-agent, hierarchical reinforcement learning system for sequential decision problems. It was created as a platform for HierGen, an algorithm for hierarchical structure discovery in sequential decision problems. For details, please refer to: Neville Mehta. Hierarchical Structure Discovery and Transfer in Sequential Decision Problems. Ph.D thesis, Oregon State University, 2011. ---------- Features ---------- * Facilitates the implementation of hierarchical and non-hierarchical learning algorithms. * Incorporates multi-agent learning. * Facilitates the implementation of sequential decision problems. -------------- Requirements -------------- (The versions that HierLearning has been verified on are mentioned in parentheses.) * Compiler: Visual Studio (2012, v11) or gcc (v4.8.1) * Weka (v3.6.5) * Python (v2.7) Optional: * Graphviz (v2.28) * Wargus (v2.1) * Octave (v3.2.4) -------------- Installation -------------- To build binary: make To clean: make clean ------- Usage ------- hierlearning -h hierlearning -d -l [-r -e ] hierlearning -d -n -t hierlearning -d -l -n [-m ] [-r -e ] hierlearning -d -l -t [-m ] [-r -e ] ---------- Examples ---------- To load the manually-designed hierarchy and execute 10 runs of 100 episodes each: hierlearning -d taxi -l maxq -r 10 -e 100 To generate 50 random trajectories: hierlearning -d taxi -n 50 -t trajectory.out To read the trajectory file and generate the task hierarchy based on the supplied models: hierlearning -d taxi -l maxq -t trajectory.out -m models To generate 50 random trajectories, build the task hierarchy, and execute 10 runs of 100 episodes each: hierlearning -d taxi -l maxq -n 50 -r 10 -e 100 ----------- Execution ----------- Run on a cluster using qsub: cluster Process the output (needs Octave): process_results hierlearning/simulator.cc0000664000175000017500000002001312303746004015726 0ustar nevilleneville /************************************************* SIMULATOR Neville Mehta **************************************************/ #include #include #include #include #include "lib/common.h" #include "simulator.h" #if defined(NDEBUG) const bool display = false; #else const bool display = true; #endif const unsigned max_steps = 100000; void Simulator::episode (MDP& mdp, Learner& learner, const unsigned& num_runs, const unsigned& run_index, const unsigned& num_episodes, const bool averaging) const { valarray avg_reward(0.0, num_episodes + 1); // Total reward through episode completion valarray avg_duration(0.0, num_episodes + 1); // Total duration through episode completion for (unsigned run = 0; run < num_runs; ++run) { if (display) cout << "\nTrial " << run + 1 << endl << "Evaluating ..."; mdp.initialize(true); learner.initialize(mdp); // Initialize the learner's innards // Initial performance without learning double total_reward = 0.0; unsigned step = 0; while (!mdp.terminated()) { vector action = learner.learned_policy(mdp.state()); // Exercise the learner's greedy policy mdp.process(action); total_reward += mdp.reward().sum(); ++step; if (step > max_steps) throw HierException(__FILE__, __LINE__, "Reached maximum number of steps."); } if (averaging) { avg_reward[0] += (total_reward - avg_reward[0])/(run + 1); avg_duration[0] += (step - avg_duration[0])/(run + 1); } else { avg_reward[0] = total_reward; avg_duration[0] = step; } for (unsigned episode = 1; episode <= num_episodes; ++episode) { if (display) cout << " " << episode; mdp.initialize(true); learner.reset(); total_reward = 0.0; step = 0; while (!mdp.terminated()) { vector action = learner.exploratory_policy(mdp.state()); // Exercise the learner's exploratory policy mdp.process(action); // Determine next state, reward, and time from the environment learner.update(mdp.state(), mdp.reward(), mdp.duration()); total_reward += mdp.reward().sum(); ++step; if (step > max_steps) throw HierException(__FILE__, __LINE__, "Reached maximum number of steps."); } if (averaging) { avg_reward[episode] += (total_reward - avg_reward[episode])/(run + 1); avg_duration[episode] += (step - avg_duration[episode])/(run + 1); } else { avg_reward[episode] = total_reward; avg_duration[episode] = step; } } if (display) cout << ".\n"; if (!averaging) { file_table(avg_reward, "results/reward_" + mdp.name() + "_" + learner.name() + "_" + to_string(run_index + run) + ".out"); file_table(avg_duration, "results/duration_" + mdp.name() + "_" + learner.name() + "_" + to_string(run_index + run) + ".out"); } } // Write the average durations to a file if (averaging) { file_table(avg_reward, "results/reward_" + mdp.name() + "_" + learner.name() + "_" + to_string(run_index) + ".out"); file_table(avg_duration, "results/duration_" + mdp.name() + "_" + learner.name() + "_" + to_string(run_index) + ".out"); } } void Simulator::step (MDP& mdp, Learner& learner, const unsigned& num_runs, const unsigned& run_index, const unsigned& num_steps, const bool averaging) const { const int averaging_interval = 100; valarray avg_reward(0.0, num_steps/averaging_interval); // Average reward through steps for (unsigned run = 0; run < num_runs; ++run) { if (display) cout << "\nTrial " << run + 1 << endl << "Evaluating ..."; mdp.initialize(true); learner.initialize(mdp); // Initialize the learner's innards double average_reward = 0.0; for (unsigned step = 0; step < num_steps; ++step) { if (display && (step + 1) % averaging_interval == 0) cout << " " << step/averaging_interval; vector action = learner.exploratory_policy(mdp.state()); // Exercise the learner's exploratory policy mdp.process(action); // Determine next state, reward, and time from the environment learner.update(mdp.state(), mdp.reward(), mdp.duration()); average_reward += (mdp.reward().sum() - average_reward)/(step + 1); // Evaluation if ((step + 1) % averaging_interval == 0) { int eval_index = step/averaging_interval; //for (unsigned eval_step = 0; eval_step < averaging_interval; ++eval_step) //{ if (mdp.terminated()) // { mdp.initialize(true); // learner.reset(); // } // vector action = learner.greedy_policy(mdp.state()); // mdp.process(action); // average_reward += (mdp.reward().sum() - average_reward)/(averaging_interval * eval_index + eval_step + 1); //} if (averaging) avg_reward[eval_index] += (average_reward - avg_reward[eval_index])/(run + 1); else avg_reward[eval_index] = average_reward; } if (mdp.terminated()) { mdp.initialize(true); learner.reset(); } } if (display) cout << ".\n"; if (!averaging) file_table(avg_reward, "results/reward_" + mdp.name() + "_" + learner.name() + "_" + to_string(run_index + run) + ".out"); } // Write the average durations to a file if (averaging) file_table(avg_reward, "results/reward_" + mdp.name() + "_" + learner.name() + "_" + to_string(run_index) + ".out"); } vector>> Simulator::trajectory_generator (const string& trajectory_filename, MDP& mdp, Learner* const learner, const unsigned& num_trajectories) const { vector>> trajectories(num_trajectories); if (num_trajectories) { vector mdp_actions = mdp.actions(); ofstream trajectory_file; if (!trajectory_filename.empty()) { size_t directory_index = trajectory_filename.find_last_of("/"); if (directory_index != string::npos) create_directory(trajectory_filename.substr(0, directory_index)); trajectory_file.open(trajectory_filename.c_str()); if (!trajectory_file.is_open()) throw HierException(__FILE__, __LINE__, "Cannot write to the trajectory file."); // Actions for (unsigned a = 0; a < mdp_actions.size(); ++a) { if (a > 0) trajectory_file << " "; trajectory_file << mdp.print_action(mdp_actions[a]); } trajectory_file << endl; // Action indices for (unsigned a = 0; a < mdp_actions.size(); ++a) { if (a > 0) trajectory_file << " "; trajectory_file << mdp_actions[a]; } trajectory_file << endl; // Variables vector state_variables = mdp.state().variables(); for (auto v = state_variables.cbegin(); v != state_variables.cend(); ++v) { if (v != state_variables.begin()) trajectory_file << " "; trajectory_file << mdp.state().variable_name(*v); } trajectory_file << endl; // Variable sizes for (auto v = state_variables.cbegin(); v != state_variables.cend(); ++v) { if (v != state_variables.begin()) trajectory_file << " "; trajectory_file << mdp.state().variable_size(*v); } trajectory_file << "\n#\n"; } if (display) cout << "Writing trajectories ..."; for (unsigned t = 0; t < num_trajectories; ++t) { if (display) cout << " " << t + 1; mdp.initialize(learner != nullptr); if (learner) learner->reset(); while (!mdp.terminated()) { vector action; if (learner) action = learner->learned_policy(mdp.state()); else { action = vector(mdp.state().num_agents()); for (unsigned a = 0; a < action.size(); ++a) if (mdp.state().action_complete(a)) action[a] = mdp_actions[rand_int(mdp_actions.size())]; } if (trajectory_file.is_open()) trajectory_file << mdp.state().print() << endl << action[0] << endl; else trajectories[t].emplace_back(new State_Action_Reward(mdp.state(), action)); mdp.process(action); if (trajectory_file.is_open()) trajectory_file << mdp.reward()[0] << endl; else trajectories[t].back()->reward = mdp.reward(); } if (trajectory_file.is_open()) trajectory_file << mdp.state().print() << "\n#\n"; else trajectories[t].emplace_back(new State_Action_Reward(mdp.state(), vector())); } if (trajectory_file.is_open()) trajectory_file.close(); if (display) cout << ".\n"; } return trajectories; } hierlearning/cluster0000664000175000017500000000154312303746004015013 0ustar nevilleneville#!/bin/sh ######################### # # # Qsub Script Creator # # Neville Mehta # # # ######################### if [ $# -lt 5 ] then echo "Usage: $0 " exit 1 fi for ((r = 1; r <= $4; ++r)); do cat > qsub.sh <" run = 1; curve = []; failed_runs = 0; for run = 1:total_runs filename = sprintf('%s_%u.out', filename_str, run); if (exist(filename, 'file')) temp = load(filename); curve(:,end+1) = temp(:,end); else ++failed_runs; end end if size(curve) > 0 filename = sprintf('%s_coalesced.out', filename_str); dlmwrite(filename, curve, '\t'); filename = sprintf('%s.out', filename_str); dlmwrite(filename, mean(curve, 2), '\t'); end printf("Success = %f%%\n", (total_runs - failed_runs)*100/total_runs); end hierlearning/learner/0000775000175000017500000000000012205065452015036 5ustar nevillenevillehierlearning/learner/hier/0000775000175000017500000000000012205065452015765 5ustar nevillenevillehierlearning/learner/hier/generic/0000775000175000017500000000000012304226346017402 5ustar nevillenevillehierlearning/learner/hier/generic/sequenced_task.cc0000664000175000017500000000141712205065452022711 0ustar nevilleneville /********************************************************************************** SEQUENCED TASK Neville Mehta ***********************************************************************************/ #include "sequenced_task.h" unsigned SequencedTask::sequenced_subtask (const unsigned& agent, const vector& parameters, const State& state) const { /* A subtask is admissible if: 1. It is primitive. OR 1. The subtask's parameters can be grounded legally. 2. The subtask is not terminated. */ for (unsigned t = _subtask_index; t < _num_subtasks; ++t) if (_subtask[t].link->admissible(agent, subtask_bindings(agent, parameters, state, t), state)) return t; throw HierException(__FILE__, __LINE__, "Task " + _name + " has no admissible child tasks."); } hierlearning/learner/hier/generic/task.cc0000664000175000017500000001244212213030214020640 0ustar nevilleneville /********************************************************************************** GENERIC TASK FRAMEWORK Neville Mehta ***********************************************************************************/ #include #include #include #include "../../../lib/common.h" #include "task.h" Task::Task (const string& task_name, const string& parameters, const string& state_variables, const MDP& mdp) : _name(task_name) { vector parameter_tokens = tokenize(parameters, ", "); // Tokenize parameter types separated by "," for (const string& str : parameter_tokens) _parameter_size.push_back(mdp.state().parse(str).second); // Determining the total number of parameter bindings _num_bindings = accumulate(_parameter_size.begin(), _parameter_size.end(), 1, multiplies()); // Tokenize state variables separated by "," vector state_variable_tokens = tokenize(state_variables, ", "); for (const string& state_variable : state_variable_tokens) { if (state_variable.substr(0, 6) == "agent_") // Agent-based _variables.emplace_back(1, mdp.state().variable_index(state_variable)); else if (state_variable.substr(0, 10) == "parameter_") // Parameter-based { int p = from_string(state_variable.substr(10)); string::size_type position; if (p >= 0 && p < int(_parameter_size.size()) && (position = state_variable.find(':', 10)) != string::npos) _variables.emplace_back(2 + p, mdp.state().variable_index(state_variable.substr(position + 1))); else throw HierException(__FILE__, __LINE__, "Incorrect specification of parameter-based state variable."); } else _variables.emplace_back(0, mdp.state().variable_index(state_variable)); } // Determining the state space size of the task _num_states = 1; for (const auto& v : _variables) _num_states *= mdp.state().variable_size(v.value); } set Task::abstraction () const { set abstraction; for (const auto& v : _variables) abstraction.insert(v.value); return abstraction; } string Task::abstraction_str () const { ostringstream out; for (auto v_itr = _variables.cbegin(); v_itr != _variables.cend(); ++v_itr) { if (v_itr != _variables.cbegin()) out << ","; switch (v_itr->type) { case 0: // Regular out << "v"; break; case 1: // Agent-based out << "av"; break; default: // Parameter-based out << "p"; } out << v_itr->value; } return out.str(); } string Task::abstraction_str_html () const { ostringstream out; for (auto v_itr = _variables.cbegin(); v_itr != _variables.cend(); ++v_itr) { if (v_itr != _variables.cbegin()) out << ","; switch (v_itr->type) { case 0: // Regular out << "v"; break; case 1: // Agent-based out << "av"; break; default: // Parameter-based out << "p"; } out << "" << v_itr->value << ""; } return out.str(); } int Task::hash (const unsigned& agent, const vector& parameters, const State& state) const { int hash = 0; // Parameters for (int p = int(parameters.size()) - 1; p >= 0; --p) hash = _parameter_size[p] * hash + parameters[p]; // State variables for (auto v_itr = _variables.crbegin(); v_itr != _variables.crend(); ++v_itr) { int var; switch (v_itr->type) { case 0: // Regular var = v_itr->value; break; case 1: // Agent-based var = agent * state.num_agent_variables() + v_itr->value; break; default: // Parameter-based var = parameters[v_itr->type - 2] + v_itr->value; } hash = state.variable_size(var) * hash + state.variable(var); } return hash; } pair, unique_ptr> Task::unhash (const unsigned& agent, int hash, const State& state) const { // Unhash parameters vector parameters(_parameter_size.size()); int par_hash = hash / _num_states; for (unsigned p = 0; p < parameters.size(); ++p) { parameters[p] = par_hash % _parameter_size[p]; par_hash /= _parameter_size[p]; } // Unhash state variables unique_ptr _state = state.copy(); for (const auto& v : _variables) { int var; switch (v.type) { case 0: // Regular var = v.value; break; case 1: // Agent-based var = agent * state.num_agent_variables() + v.value; break; default: // Parameter-based var = parameters[v.type - 2] + v.value; } _state->variable(var) = hash % state.variable_size(var); hash /= state.variable_size(var); } return make_pair(parameters, move(_state)); } bool Task::update_parameters (const vector& parameter_size) { if (parameter_size.size() != _parameter_size.size()) throw HierException(__FILE__, __LINE__, "Differing number of parameters."); // Assign the larger size bool updated = false; for (unsigned p = 0; p < _parameter_size.size(); ++p) if (_parameter_size[p] < parameter_size[p]) { _parameter_size[p] = parameter_size[p]; updated = true; } if (updated) _num_bindings = accumulate(_parameter_size.begin(), _parameter_size.end(), 1, multiplies()); return updated; } string Task::print_dot () const { ostringstream output; output << name() << " [label=<" << name(); if (num_parameters()) { output << "("; for (unsigned p = 0; p < num_parameters(); ++p) { if (p > 0) output << ","; output << "x" << p << ""; } output << ")"; } output << "
"; return output.str(); } hierlearning/learner/hier/generic/hier_learner.h0000664000175000017500000000363112213215430022204 0ustar nevilleneville /************************************************* HIERARCHICAL LEARNER Neville Mehta **************************************************/ #pragma once #include #include #include "../../../domain/mdp.h" #include "../../../hiergen/hierarchy.h" #include "task.h" #include "../../learner.h" class HierLearner : public Learner { private: // Task stack for controlled recursion struct Task_stack { Task* task; // Pointer to task module vector parameters; // Parameter bindings for the task unique_ptr state; // World state when the subtask is invoked unsigned subtask; // Index of subtask double time; // Time to execute the task (for discounted learners) Task_stack (Task* const task, const vector& parameters, const double& time = 0.0) : task(task), parameters(parameters), time(time) {} ~Task_stack () {} }; protected: Task* Root_task; list Task_list; // List of all tasks vector> task_stack; // Front of the list = top of stack (Can't use STL stack because we need to check the entire stack for task termination) void create_task_list(); public: HierLearner(const string& learner_name, const MDP& mdp); template void flat_hierarchy_designer(const MDP& mdp); // Composite task, Primitive task, Root task template void task_hierarchy_designer(const string& type, const MDP& mdp); virtual void initialize(const MDP& mdp); virtual void reset(); virtual vector greedy_policy(const State& state); virtual vector exploratory_policy(const State& state); virtual vector learned_policy(const State& state); virtual void print_hierarchy(const string& filename) const; virtual void write_text_file(const string& filename, const MDP& mdp) const; virtual ~HierLearner(); }; #include "hier_learner.inl" hierlearning/learner/hier/generic/composite_task.cc0000664000175000017500000002025112205065452022734 0ustar nevilleneville /********************************************************************************** GENERIC COMPOSITE TASK Neville Mehta ***********************************************************************************/ #include #include "../../../lib/common.h" #include "composite_task.h" CompositeTask::CompositeTask (const string& task_name, const string& parameters, const string& state_variables, const vector& subtasks, bool (*function)(const unsigned& agent, const vector& parameters, const State& state), const Expression& expression, const MDP& mdp) : Task(task_name, parameters, state_variables, mdp), _num_subtasks(0), _termination_function(function), _termination_expression(expression) { for (const auto& subtask : subtasks) add_subtask(subtask, mdp); // Resize all together at the end } void CompositeTask::add_subtask (const Subtask& subtask, const MDP& mdp) { vector subtask_parameters_str = tokenize(subtask.parameters, ", "); // Tokenize the subtask's parameter binding string if (subtask_parameters_str.size() == subtask.link->_parameter_size.size()) // #(bindings) = #(parameters) { vector subtask_parameters(subtask_parameters_str.size()); vector free_parameters; for (unsigned p = 0; p < subtask_parameters_str.size(); ++p) { if (subtask_parameters_str[p] == "?") // Actual parameter binding value stored in the 'value' field { subtask_parameters[p].type = 0; subtask_parameters[p].value = 0; free_parameters.push_back(p); } else if (subtask_parameters_str[p].substr(0, 9) == "constant:") // Parameter binding value is constant = 'value' { subtask_parameters[p].type = 0; subtask_parameters[p].value = mdp.state().parse(subtask_parameters_str[p].substr(9, subtask_parameters_str[p].find_first_of("+- "))).second; } else if (subtask_parameters_str[p].substr(0, 10) == "parameter:") // Parameter binding value comes from the parent task's parameter indexed by 'value' { subtask_parameters[p].type = 1; istringstream par(subtask_parameters_str[p].substr(10)); if (!(par >> subtask_parameters[p].value) || subtask_parameters[p].value < 0 || subtask_parameters[p].value >= int(_parameter_size.size())) throw HierException(__FILE__, __LINE__, "Unknown parameter specification."); } else if (subtask_parameters_str[p].substr(0, 6) == "agent_") // Parameter binding value comes from the agent's state variable indexed by 'value' { subtask_parameters[p].type = 2; subtask_parameters[p].value = mdp.state().variable_index(subtask_parameters_str[p].substr(0, subtask_parameters_str[p].find_first_of("+- "))); } else // Parameter binding value comes from the state variable indexed by 'value' { subtask_parameters[p].type = 3; subtask_parameters[p].value = mdp.state().variable_index(subtask_parameters_str[p].substr(0, subtask_parameters_str[p].find_first_of("+- "))); } // Offset subtask_parameters[p].offset = (subtask_parameters_str[p].find_first_of("+-") != string::npos) ? mdp.state().parse(subtask_parameters_str[p].substr(subtask_parameters_str[p].find_first_of("+-"))).second : 0; } _subtask.emplace_back(subtask.link, subtask_parameters); ++_num_subtasks; // All possible bindings permutations for the free subtask parameters while (free_parameters.size()) // Free parameters exist { unsigned fp = 0; ++subtask_parameters[free_parameters[fp]].value; while (subtask_parameters[free_parameters[fp]].value == subtask.link->_parameter_size[free_parameters[fp]] && fp < free_parameters.size() - 1) { subtask_parameters[free_parameters[fp]].value = 0; // Reset its value ++subtask_parameters[free_parameters[++fp]].value; // Increment the next free parameter } if (subtask_parameters[free_parameters.back()].value == subtask.link->_parameter_size[free_parameters.back()]) break; // All permutations have been exhausted // Insert new parameter combo into the subtask data structure _subtask.emplace_back(subtask.link, subtask_parameters); ++_num_subtasks; } } else throw HierException(__FILE__, __LINE__, "The parameter and binding lists aren't of the same size."); } vector CompositeTask::subtask_bindings (const unsigned& agent, const vector& parameters, const State& state, const unsigned& t) const { vector subtask_bindings(_subtask[t].parameters.size()); for (unsigned p = 0; p < subtask_bindings.size(); ++p) { switch (_subtask[t].parameters[p].type) { case 0: // Actual value subtask_bindings[p] = _subtask[t].parameters[p].value; break; case 1: // Parameter value subtask_bindings[p] = parameters[_subtask[t].parameters[p].value]; break; case 2: // Agent-dependent state variable subtask_bindings[p] = state.variable(agent * state.num_agent_variables() + _subtask[t].parameters[p].value); break; case 3: // State variable subtask_bindings[p] = state.variable(_subtask[t].parameters[p].value); break; } subtask_bindings[p] += _subtask[t].parameters[p].offset; if (subtask_bindings[p] < 0 || subtask_bindings[p] >= _subtask[t].link->_parameter_size[p]) throw ParameterException(__FILE__, __LINE__, "Parameter binding out of range."); } return subtask_bindings; } string CompositeTask::binding_str (const unsigned& t) const { if (_subtask[t].parameters.empty()) return ""; ostringstream out; for (unsigned p = 0; p < _subtask[t].parameters.size(); ++p) { if (p > 0) out << ","; switch (_subtask[t].parameters[p].type) { case 0: // Actual value break; case 1: // Parameter value out << "x"; break; case 2: // Agent-dependent state variable out << "av"; break; case 3: // State variable out << "v"; break; } out << _subtask[t].parameters[p].value; } return out.str(); } string CompositeTask::binding_str_html (const unsigned& t) const { if (_subtask[t].parameters.empty()) return ""; ostringstream out; for (unsigned p = 0; p < _subtask[t].parameters.size(); ++p) { if (p > 0) out << ","; switch (_subtask[t].parameters[p].type) { case 0: // Actual value break; case 1: // Parameter value out << "x"; break; case 2: // Agent-dependent state variable out << "av"; break; case 3: // State variable out << "v"; break; } out << "" << _subtask[t].parameters[p].value << ""; } return out.str(); } vector CompositeTask::admissible_subtasks (const unsigned& agent, const vector& parameters, const State& state) const { vector adm_tasks; /* A subtask is admissible if: 1. It is primitive. OR 1. The subtask's parameters can be grounded legally. 2. The subtask is not terminated. */ for (unsigned t = 0; t < _num_subtasks; ++t) if (_subtask[t].link->admissible(agent, subtask_bindings(agent, parameters, state, t), state)) adm_tasks.push_back(t); if (!adm_tasks.empty()) return adm_tasks; throw HierException(__FILE__, __LINE__, _name + " has no admissible children."); } unsigned CompositeTask::exploratory_policy (const unsigned& agent, const vector& parameters, const State& state) { vector adm_tasks = admissible_subtasks(agent, parameters, state); unsigned num_admissible_tasks = adm_tasks.size(); if (num_admissible_tasks == 1) return adm_tasks[0]; if (rand_real() < composite_task_parameters::EPSILON) // Random return adm_tasks[rand_int(num_admissible_tasks)]; else return greedy_policy(agent, parameters, state); } bool CompositeTask::terminated (const unsigned& agent, const vector& parameters, const State& state) const { if (_termination_function) return _termination_function(agent, parameters, state); else return _termination_expression.evaluate(state, parameters, agent) != 0; } string CompositeTask::print_dot () const { string output = Task::print_dot() + termination_expression().print_html() + "
{" + abstraction_str_html() + "}>]\n"; // Child tasks for (unsigned t = 0; t < num_subtasks(); ++t) { output += name() + " -> " + get_subtask(t)->name(); string binding_str = binding_str_html(t); if (!binding_str.empty()) output += " [label=<" + binding_str + ">,fontsize=10]"; output += "\n"; } return output; } hierlearning/learner/hier/generic/hier_learner.inl0000664000175000017500000002543412213215646022555 0ustar nevilleneville /************************************************* HIERARCHICAL LEARNER Neville Mehta **************************************************/ #pragma once #include "../../../domain/domains.h" inline bool Root_termination (const unsigned& agent, const vector& parameters, const State& state) { return false; } template void HierLearner::flat_hierarchy_designer (const MDP& mdp) { // Create the shallowest hierarchy (root task with primitive children) without smart abstraction if (mdp.name().substr(0,4) == "taxi") { vector move; move.push_back(new PTask("North", "agent_x, agent_y, passenger_location_x, passenger_location_y, passenger_destination_x, passenger_destination_y, passenger_in_taxi", Taxi::North, mdp)); move.push_back(new PTask("South", "agent_x, agent_y, passenger_location_x, passenger_location_y, passenger_destination_x, passenger_destination_y, passenger_in_taxi", Taxi::South, mdp)); move.push_back(new PTask("East", "agent_x, agent_y, passenger_location_x, passenger_location_y, passenger_destination_x, passenger_destination_y, passenger_in_taxi", Taxi::East, mdp)); move.push_back(new PTask("West", "agent_x, agent_y, passenger_location_x, passenger_location_y, passenger_destination_x, passenger_destination_y, passenger_in_taxi", Taxi::West, mdp)); Task* pickup = new PTask("Pickup", "agent_x, agent_y, passenger_location_x, passenger_location_y, passenger_destination_x, passenger_destination_y, passenger_in_taxi", Taxi::Pickup, mdp); Task* dropoff = new PTask("Dropoff", "agent_x, agent_y, passenger_location_x, passenger_location_y, passenger_destination_x, passenger_destination_y, passenger_in_taxi", Taxi::Dropoff, mdp); // Root task vector subtasks(move); // Attach primitive move tasks subtasks.push_back(pickup); subtasks.push_back(dropoff); Root_task = new RTask("Taxi_Task", "", "agent_x, agent_y, passenger_location_x, passenger_location_y, passenger_destination_x, passenger_destination_y, passenger_in_taxi", subtasks, &Root_termination, Expression(), mdp); } else if (mdp.name().substr(0,7) == "bitflip") { vector mdp_actions = mdp.actions(); string state_variables; for (unsigned a = 0; a < mdp_actions.size(); ++a) { if (a > 0) state_variables += ", "; state_variables += "bit_" + to_string(mdp_actions[a]); } vector subtasks; for (const auto& mdp_action : mdp_actions) subtasks.push_back(new PTask("Flip_" + to_string(mdp_action), state_variables, mdp_action, mdp)); Root_task = new RTask("Bitflip_Task", "", state_variables, subtasks, &Root_termination, Expression(), mdp); } else if (mdp.name().substr(0,6) == "wargus") { // Navigation vector move; Task* mine_gold = nullptr, * chop_wood = nullptr, * deposit = nullptr; if (mdp.name().substr(6) == "_goto") { const Wargus_State& state = static_cast(mdp.state()); move.resize(state.regions.size()); for (unsigned site = 0; site < move.size(); ++site) { move[site].link = new PTask(string("Goto") + "_" + to_string(state.regions.data()[site].coord.x) + "_" + to_string(state.regions.data()[site].coord.y), "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_Goto::Move + state.regions.data()[site].coord.x + state.regions.data()[site].coord.y * state.map_size().x, mdp); } mine_gold = new PTask("MineGold", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_Goto::MineGold, mdp); chop_wood = new PTask("ChopWood", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_Goto::ChopWood, mdp); deposit = new PTask("Deposit", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_Goto::Deposit, mdp); } else if (mdp.name().substr(6) == "_nsew") { move.push_back(new PTask("North", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_NSEW::North, mdp)); move.push_back(new PTask("South", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_NSEW::South, mdp)); move.push_back(new PTask("East", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_NSEW::East, mdp)); move.push_back(new PTask("West", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_NSEW::West, mdp)); mine_gold = new PTask("MineGold", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_NSEW::MineGold, mdp); chop_wood = new PTask("ChopWood", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_NSEW::ChopWood, mdp); deposit = new PTask("Deposit", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", Wargus_NSEW::Deposit, mdp); } // Root task vector subtasks(move); subtasks.push_back(mine_gold); subtasks.push_back(chop_wood); subtasks.push_back(deposit); Root_task = new RTask("Resource_Gathering_Task", "", "agent_x, agent_y, agent_resource, agent_region_gold, agent_region_wood, agent_region_townhall, requisite_gold, requisite_wood", subtasks, &Root_termination, Expression(), mdp); } else throw HierException(__FILE__, __LINE__, "Unknown MDP."); } inline bool Goto_termination (const unsigned& agent, const vector& parameters, const State& state) { return parameters[0] == state.variable(agent * state.num_agent_variables() + state.variable_index("agent_x")) && parameters[1] == state.variable(agent * state.num_agent_variables() + state.variable_index("agent_y")); } inline bool Get_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(state.variable_index("passenger_in_taxi")) == 1; } inline bool Put_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(state.variable_index("passenger_in_taxi")) == 0; } inline bool Get_Goto_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(agent * state.num_agent_variables() + state.variable_index("agent_x")) == state.variable(state.variable_index("passenger_location_x")) && state.variable(agent * state.num_agent_variables() + state.variable_index("agent_y")) == state.variable(state.variable_index("passenger_location_y")); } inline bool Put_Goto_termination (const unsigned& agent, const vector& parameters, const State& state) { return state.variable(agent * state.num_agent_variables() + state.variable_index("agent_x")) == state.variable(state.variable_index("passenger_destination_x")) && state.variable(agent * state.num_agent_variables() + state.variable_index("agent_y")) == state.variable(state.variable_index("passenger_destination_y")); } template void HierLearner::task_hierarchy_designer (const string& type, const MDP& mdp) { if (type == "taxi:manual") { vector move; move.push_back(new PTask("North", "", Taxi::North, mdp)); move.push_back(new PTask("South", "", Taxi::South, mdp)); move.push_back(new PTask("East", "", Taxi::East, mdp)); move.push_back(new PTask("West", "", Taxi::West, mdp)); Task* pickup = new PTask("Pickup", "agent_x, agent_y, passenger_location_x, passenger_location_y", Taxi::Pickup, mdp); Task* dropoff = new PTask("Dropoff", "agent_x, agent_y, passenger_destination_x, passenger_destination_y", Taxi::Dropoff, mdp); Task* Goto = new CTask("Goto", "map_size.x, map_size.y", "agent_x, agent_y", move, Goto_termination, Expression(), mdp); vector subtasks; subtasks.emplace_back(Goto, "passenger_location_x, passenger_location_y"); subtasks.push_back(pickup); Task* Get = new CTask("Get", "", "agent_x, agent_y, passenger_location_x, passenger_location_y", subtasks, Get_termination, Expression(), mdp); subtasks.clear(); subtasks.emplace_back(Goto, "passenger_destination_x, passenger_destination_y"); subtasks.push_back(dropoff); Task* Put = new CTask("Put", "", "agent_x, agent_y, passenger_destination_x, passenger_destination_y", subtasks, Put_termination, Expression(), mdp); // Root task subtasks.clear(); subtasks.push_back(Get); subtasks.push_back(Put); Root_task = new RTask("Taxi_Task", "", "agent_x, agent_y, passenger_location_x, passenger_location_y, passenger_destination_x, passenger_destination_y, passenger_in_taxi", subtasks, &Root_termination, Expression(), mdp); } else if (type == "taxi:hi-mat") { vector move; move.push_back(new PTask("North", "", Taxi::North, mdp)); move.push_back(new PTask("South", "", Taxi::South, mdp)); move.push_back(new PTask("East", "", Taxi::East, mdp)); move.push_back(new PTask("West", "", Taxi::West, mdp)); Task* pickup = new PTask("Pickup", "agent_x, agent_y, passenger_location_x, passenger_location_y", Taxi::Pickup, mdp); Task* dropoff = new PTask("Dropoff", "agent_x, agent_y, passenger_destination_x, passenger_destination_y", Taxi::Dropoff, mdp); Task* Get_Goto = new CTask("Get_Goto", "", "agent_x, agent_y, passenger_location_x, passenger_location_y", move, Get_Goto_termination, Expression(), mdp); Task* Put_Goto = new CTask("Put_Goto", "", "agent_x, agent_y, passenger_destination_x, passenger_destination_y", move, Put_Goto_termination, Expression(), mdp); vector subtasks; subtasks.push_back(Get_Goto); subtasks.push_back(pickup); Task* Get = new CTask("Get", "", "agent_x, agent_y, passenger_location_x, passenger_location_y", subtasks, Get_termination, Expression(), mdp); subtasks.clear(); subtasks.push_back(Put_Goto); subtasks.push_back(dropoff); Task* Put = new CTask("Put", "", "agent_x, agent_y, passenger_destination_x, passenger_destination_y", subtasks, Put_termination, Expression(), mdp); // Root task subtasks.clear(); subtasks.push_back(Get); subtasks.push_back(Put); Root_task = new RTask("Taxi_Task", "", "agent_x, agent_y, passenger_location_x, passenger_location_y, passenger_destination_x, passenger_destination_y, passenger_in_taxi", subtasks, &Root_termination, Expression(), mdp); } else throw HierException(__FILE__, __LINE__, "Hierarchy type \"" + type + "\" does not exist."); } hierlearning/learner/hier/generic/sequenced_task.h0000664000175000017500000000275112205065452022555 0ustar nevilleneville /********************************************************************************** SEQUENCED TASK Neville Mehta ***********************************************************************************/ #pragma once #include #include #include "composite_task.h" class SequencedTask : public CompositeTask { unsigned _subtask_index; unsigned sequenced_subtask(const unsigned& agent, const vector& parameters, const State& state) const; public: SequencedTask(const string& task_name, const string& parameters, const string& state_variables, const vector& subtasks, bool (*function)(const unsigned& agent, const vector& parameters, const State& state), const Expression& expression, const MDP& mdp) : Task(task_name, parameters, state_variables, mdp), CompositeTask(task_name, parameters, state_variables, subtasks, function, expression, mdp), _subtask_index(0) {} unsigned greedy_policy (const unsigned& agent, const vector& parameters, const State& state) const { return sequenced_subtask(agent, parameters, state); } unsigned exploratory_policy (const unsigned& agent, const vector& parameters, const State& state) const { return sequenced_subtask(agent, parameters, state); } void update (const int& subtask_index) { _subtask_index = subtask_index + 1; } bool termination (const unsigned& agent, const vector& parameters, const State& state) const { return _subtask_index >= _num_subtasks; } ~SequencedTask () {} }; hierlearning/learner/hier/generic/task.h0000664000175000017500000000363412205065452020522 0ustar nevilleneville /********************************************************************************** GENERIC TASK FRAMEWORK Neville Mehta ***********************************************************************************/ #pragma once #include #include #include #include #include "../../../domain/mdp.h" class Task { protected: struct Variable { int type; // Indicates the context of the 'value' field: type = 0: value indexes the state variable; type >= 1: value indexes the parameter-based state variable int value; Variable (const int& type, const int& value) : type(type), value(value) {} }; const string _name; // Name of task unsigned _num_bindings; // Total number of parameter binding combinations list _variables; // Task's state variables unsigned _num_states; // Number of states public: vector _parameter_size; // Task parameters Task(const string& task_name, const string& parameters, const string& state_variables, const MDP& mdp); string name () const { return _name; } virtual bool primitive() const = 0; virtual bool admissible(const unsigned& agent, const vector& parameters, const State& state) const = 0; virtual void initialize() = 0; set abstraction() const; string abstraction_str() const; string abstraction_str_html() const; int hash(const unsigned& agent, const vector& parameters, const State& state) const; pair, unique_ptr> unhash(const unsigned& agent, int hash, const State& state) const; unsigned num_parameters () const { return _parameter_size.size(); } virtual bool update_parameters(const vector& parameter_size); virtual bool terminated(const unsigned& agent, const vector& parameters, const State& state) const = 0; virtual string print(const State& state) const = 0; virtual string print_dot() const; // Graphviz dot format virtual ~Task () {} }; hierlearning/learner/hier/generic/composite_task.h0000664000175000017500000000644612205065452022610 0ustar nevilleneville /********************************************************************************** GENERIC COMPOSITE TASK Neville Mehta ***********************************************************************************/ #pragma once #include #include #include "../../../lib/matrix.h" #include "../../../hiergen/expression.h" #include "task.h" namespace composite_task_parameters { const double EPSILON = 0.08; // Exploration factor } class CompositeTask : public virtual Task { struct Parameter { int type; int value; /* The 'type' field indicates the context of the 'value' field: type = 0: value contains the actual binding value; type = 1: value indexes the parent's parameter vector; type = 2: value indexes the state variable. */ int offset; // Offsets the net value }; struct SubTask // Local data structure { Task* link; vector parameters; SubTask (Task* link, const vector parameters) : link(link), parameters(parameters) {} }; protected: unsigned _num_subtasks; // Number of subtasks vector _subtask; // Vector of pointers to subtasks with their associated bindings bool (*_termination_function)(const unsigned& agent, const vector& parameters, const State& state); Expression _termination_expression; vector admissible_subtasks(const unsigned& agent, const vector& parameters, const State& state) const; public: struct Subtask // Input format { Task* link; string parameters; Subtask (Task* T = nullptr, const string& parameters = "") : link(T), parameters(parameters) {} bool operator == (const Subtask& subtask) const { return link == subtask.link && parameters == subtask.parameters; } }; CompositeTask(const string& task_name, const string& parameters, const string& state_variables, const vector& subtasks, bool (*function)(const unsigned& agent, const vector& parameters, const State& state), const Expression& expression, const MDP& mdp); bool primitive () const { return false; } bool admissible (const unsigned& agent, const vector& parameters, const State& state) const { return !terminated(agent, parameters, state); } unsigned num_subtasks () const { return _num_subtasks; } Task* get_subtask (const unsigned& t) const { return _subtask[t].link; } vector subtask_bindings(const unsigned& agent, const vector& parameters, const State& state, const unsigned& t) const; string binding_str(const unsigned& t) const; string binding_str_html(const unsigned& t) const; virtual void add_subtask (const Subtask& subtask, const MDP& mdp); virtual unsigned greedy_policy(const unsigned& agent, const vector& parameters, const State& state) = 0; virtual unsigned exploratory_policy(const unsigned& agent, const vector& parameters, const State& state); virtual bool terminated(const unsigned& agent, const vector& parameters, const State& state) const; Expression termination_expression () const { return _termination_expression; } virtual string print_dot() const; virtual ~CompositeTask() {} }; class ParameterException : public HierException { public: ParameterException (const string& filename, const int& line, const string& error) : HierException(filename, line, error) {} virtual ~ParameterException () throw() {} }; hierlearning/learner/hier/generic/primitive_task.h0000664000175000017500000000201212205065452022577 0ustar nevilleneville /********************************************************************************** GENERIC PRIMITIVE TASK Neville Mehta ***********************************************************************************/ #pragma once #include #include "task.h" class PrimTask : public virtual Task { protected: const int _primitive_action; public: PrimTask(const string& task_name, const string& state_variables, const int& action, const MDP& mdp) : Task(task_name, "", state_variables, mdp), _primitive_action(action) {} bool primitive () const { return true; } int primitive_action () const { return _primitive_action; } bool admissible (const unsigned& agent, const vector& parameters, const State& state) const { return true; } bool terminated (const unsigned& agent, const vector& parameters, const State& state) const { return true; } virtual string print_dot() const { return Task::print_dot() + "{" + abstraction_str_html() + "}>,shape=ellipse]\n"; } virtual ~PrimTask() {} }; hierlearning/learner/hier/generic/module.make0000664000175000017500000000020512205065452021522 0ustar nevillenevilleFOLDER = learner/hier/generic OBJ = task.o composite_task.o sequenced_task.o hier_learner.o FILES += $(addprefix $(FOLDER)/, $(OBJ)) hierlearning/learner/hier/generic/hier_learner.cc0000664000175000017500000002004312213033500022332 0ustar nevilleneville /***************************************************************************************** HIERARCHICAL LEARNER Neville Mehta ******************************************************************************************/ #include #include #include #include #include "../../../domain/domains.h" #include "primitive_task.h" #include "hier_learner.h" HierLearner::HierLearner (const string& learner_name, const MDP& mdp) : Learner(learner_name) { task_stack.resize(mdp.state().num_agents()); // One task stack per agent } void HierLearner::create_task_list () { list open_tasks(1, Root_task), primitive_tasks; while (!open_tasks.empty()) { CompositeTask* composite_task = dynamic_cast(open_tasks.front()); for (unsigned t = 0; t < composite_task->num_subtasks(); ++t) { Task* child_task = composite_task->get_subtask(t); if (find(Task_list.begin(), Task_list.end(), child_task) != Task_list.end() || find(primitive_tasks.begin(), primitive_tasks.end(), child_task) != primitive_tasks.end() || find(open_tasks.begin(), open_tasks.end(), child_task) != open_tasks.end()) continue; if (child_task->primitive()) primitive_tasks.push_back(child_task); else open_tasks.push_back(child_task); } Task_list.push_back(composite_task); open_tasks.pop_front(); } Task_list.insert(Task_list.end(), primitive_tasks.begin(), primitive_tasks.end()); } void HierLearner::initialize (const MDP& mdp) { // Initialize the agent's innards if (Task_list.empty()) create_task_list(); task_stack.resize(mdp.state().num_agents()); // One task stack per agent reset(); for_each(Task_list.begin(), Task_list.end(), [](Task* task){ task->initialize(); }); } void HierLearner::reset () { // Empty the stack(s) of everything but the Root task(s) for (auto& agent_task_stack : task_stack) { agent_task_stack.clear(); agent_task_stack.emplace_front(Root_task, vector()); // Push the Root task with no bindings } } vector HierLearner::greedy_policy (const State& state) { vector actions(state.num_agents(), -1); for (unsigned agent = 0; agent < state.num_agents(); ++agent) if (state.action_complete(agent) && !task_stack[agent].empty()) { // Finding the oldest ancester that has terminated; we start from the bottom (back) of the stack (list) auto task_itr = task_stack[agent].rbegin(); for ( ; task_itr != task_stack[agent].rend() && !task_itr->task->terminated(agent, task_itr->parameters, state); ++task_itr); if (task_itr != task_stack[agent].rend()) // Removing this oldest terminated ancestor and all of its children from the task stack task_stack[agent].erase(task_stack[agent].begin(), task_itr.base()); if (task_stack[agent].empty()) continue; while (!task_stack[agent].front().task->primitive()) { task_stack[agent].front().state = state.clone(); // Store current state info on task stack CompositeTask* composite_task = dynamic_cast(task_stack[agent].front().task); if (!composite_task) throw HierException(__FILE__, __LINE__, "Composite task expected."); task_stack[agent].front().subtask = composite_task->greedy_policy(agent, task_stack[agent].front().parameters, state); // Push new subtask and initialize its running time to 0 task_stack[agent].emplace_front(composite_task->get_subtask(task_stack[agent].front().subtask), composite_task->subtask_bindings(agent, task_stack[agent].front().parameters, state, task_stack[agent].front().subtask)); } task_stack[agent].front().state = state.clone(); actions[agent] = dynamic_cast(task_stack[agent].front().task)->primitive_action(); } return actions; } vector HierLearner::exploratory_policy (const State& state) { vector actions(state.num_agents(), -1); for (unsigned agent = 0; agent < state.num_agents(); ++agent) if (state.action_complete(agent) && !task_stack[agent].empty()) { // Finding the oldest ancester that has terminated; we start from the bottom (back) of the stack (list) auto task_itr = task_stack[agent].rbegin(); for ( ; task_itr != task_stack[agent].rend() && !task_itr->task->terminated(agent, task_itr->parameters, state); ++task_itr); if (task_itr != task_stack[agent].rend()) // Removing this oldest terminated ancestor and all of its children from the task stack task_stack[agent].erase(task_stack[agent].begin(), task_itr.base()); if (task_stack[agent].empty()) continue; while (!task_stack[agent].front().task->primitive()) { task_stack[agent].front().state = state.clone(); // Store current state info on task stack CompositeTask* composite_task = dynamic_cast(task_stack[agent].front().task); if (!composite_task) throw HierException(__FILE__, __LINE__, "Composite task expected."); task_stack[agent].front().subtask = composite_task->exploratory_policy(agent, task_stack[agent].front().parameters, state); // Push new subtask and initialize its running time to 0 task_stack[agent].emplace_front(composite_task->get_subtask(task_stack[agent].front().subtask), composite_task->subtask_bindings(agent, task_stack[agent].front().parameters, state, task_stack[agent].front().subtask)); } task_stack[agent].front().state = state.clone(); actions[agent] = dynamic_cast(task_stack[agent].front().task)->primitive_action(); } return actions; } vector HierLearner::learned_policy (const State& state) { vector actions(state.num_agents(), -1); for (unsigned agent = 0; agent < state.num_agents(); ++agent) if (state.action_complete(agent) && !task_stack[agent].empty()) { // Finding the oldest ancester that has terminated; we start from the bottom (back) of the stack (list) auto task_itr = task_stack[agent].rbegin(); for ( ; task_itr != task_stack[agent].rend() && !task_itr->task->terminated(agent, task_itr->parameters, state); ++task_itr); if (task_itr != task_stack[agent].rend()) // Removing this oldest terminated ancestor and all of its children from the task stack task_stack[agent].erase(task_stack[agent].begin(), task_itr.base()); if (task_stack[agent].empty()) continue; while (!task_stack[agent].front().task->primitive()) { CompositeTask* composite_task = dynamic_cast(task_stack[agent].front().task); if (!composite_task) throw HierException(__FILE__, __LINE__, "Composite task expected."); // Selecting the subtask greedily task_stack[agent].front().subtask = composite_task->greedy_policy(agent, task_stack[agent].front().parameters, state); // Placing the new subtask at top-of-stack task_stack[agent].emplace_front(composite_task->get_subtask(task_stack[agent].front().subtask), composite_task->subtask_bindings(agent, task_stack[agent].front().parameters, state, task_stack[agent].front().subtask)); } actions[agent] = dynamic_cast(task_stack[agent].front().task)->primitive_action(); } return actions; } void HierLearner::print_hierarchy (const string& filename) const { ofstream output((filename + ".gv").c_str()); if (!output.is_open()) throw HierException(__FILE__, __LINE__, "Unable to write hierarchy to graphviz file."); output << "digraph hierarchy {\n"; output << "node [shape=rectangle]\n"; output << "edge [arrowhead=empty]\n\n"; for (const auto& task : Task_list) output << task->print_dot() << endl; output << "}\n"; output.close(); if (system(("dot -Tpdf -o " + filename + ".pdf " + filename + ".gv").c_str())) cerr << "(" + to_string(__FILE__) + ":" + to_string(__LINE__) + ")\n"; } void HierLearner::write_text_file (const string& filename, const MDP& mdp) const { ofstream output(filename.c_str()); if (!output.is_open()) throw HierException(__FILE__, __LINE__, "Unable to write to file: " + filename); output << "MDP: " << mdp.name() << endl; // Task value functions const State& state = mdp.state(); for (const auto& task : Task_list) output << task->print(state) << endl; output.close(); } HierLearner::~HierLearner () { for (auto& task : Task_list) delete task; } hierlearning/learner/hier/hh/0000775000175000017500000000000012304226346016365 5ustar nevillenevillehierlearning/learner/hier/hh/hh_task.cc0000664000175000017500000000114612205065452020316 0ustar nevilleneville /*********************************************************************** HIERARCHICAL H TASK Neville Mehta ************************************************************************/ #include "hh_task.h" HH_Task::HH_Task (const string& task_name, const string& parameters, const string& state_variables, const MDP& mdp) : Task(task_name, parameters, state_variables, mdp) { // Create the H(s) and Time(s) tables _H.resize(_num_bindings * _num_states); _Time.resize(_num_bindings * _num_states); } void HH_Task::initialize () { // Initializing the H(s) and Time(s) tables _H = 0.0; _Time = 0.0; } hierlearning/learner/hier/hh/hh_primitive_task.cc0000664000175000017500000000300212277302520022376 0ustar nevilleneville /*********************************************************************** HIERARCHICAL H PRIMITIVE TASK Neville Mehta ************************************************************************/ #include #include "hh_primitive_task.h" HH_PrimTask::HH_PrimTask (const string& task_name, const string& state_variables, const int& action, const MDP& mdp) : Task(task_name, "", state_variables, mdp), PrimTask(task_name, state_variables, action, mdp), HH_Task(task_name, "", state_variables, mdp) { _N.resize(_num_states); } void HH_PrimTask::initialize () { HH_Task::initialize(); _N = 0; } void HH_PrimTask::update (const unsigned& agent, const State& state, const double& reward, const double& elapsed_time) { int s = hash(agent, vector(), state); _N[s]++; _Time[s] += (elapsed_time - _Time[s]) / _N[s]; _H[s] += (reward - _H[s]) / _N[s]; } string HH_PrimTask::print (const State& state) const { ostringstream out; out << "\n\n******** " << _name << " ********\n"; out << "\nReward(s):\n"; for (unsigned s = 0; s < _num_states; s++) if (_H[s] != 0.0) out << "[" << unhash(0, s, state).second->print() << "] " << _H[s] << endl; out << "\nTime(s):\n"; for (unsigned s = 0; s < _num_states; s++) if (_Time[s] != 0.0) out << "[" << unhash(0, s, state).second->print() << "] " << _Time[s] << endl; out << "\nN(s):\n"; for (unsigned s = 0; s < _num_states; s++) if (_N[s] > 0) out << "[" << unhash(0, s, state).second->print() << "] " << _N[s] << endl; return out.str(); } hierlearning/learner/hier/hh/hh_root_task.h0000664000175000017500000000251212205065452021221 0ustar nevilleneville /*********************************************************************** HIERARCHICAL H ROOT TASK Neville Mehta ************************************************************************/ #pragma once #include #include #include "hh_composite_task.h" class HH_RootTask : public HH_CompositeTask { private: valarray _greedy_subtask; valarray _alpha; double avg_adjusted_update(const unsigned& agent, const vector& parameters, const State& state); public: HH_RootTask (const string& task_name, const string& parameters, const string& state_variables, const vector& subtasks, bool (*func)(const unsigned& agent, const vector& parameters, const State& state), const Expression& expression, const MDP& mdp); void initialize(); unsigned greedy_policy(const unsigned& agent, const vector& parameters, const State& state); unsigned exploratory_policy(const unsigned& agent, const vector& parameters, const State& state); void update(const unsigned& agent, const vector& parameters, const State& state, const unsigned& t, const State& next_state); string print (const State& state) const { return "\nAverage Reward = " + to_string(*_avg_reward) + "\nAverage Time = " + to_string(*_avg_time) + "\n" + HH_CompositeTask::print(state); } ~HH_RootTask () {} }; hierlearning/learner/hier/hh/hh.cc0000664000175000017500000000574412277301114017301 0ustar nevilleneville /********************************************************************************************** HIERARCHICAL H LEARNER (with TOTAL REWARD DECOMPOSITION) Neville Mehta ***********************************************************************************************/ #include #include #include #include "hh_composite_task.h" #include "hh_root_task.h" #include "hh_primitive_task.h" #include "hh.h" valarray* HH_CompositeTask::_avg_reward; valarray* HH_CompositeTask::_avg_time; HH::HH (const string& learner_name, const MDP& mdp, const unique_ptr& hierarchy, const unsigned& run_index) : HierLearner(learner_name, mdp) { HH_CompositeTask::set_rho(&avg_reward, &avg_time); if (hierarchy) hierarchy->task_hierarchy_builder(mdp, Task_list); else flat_hierarchy_designer(mdp); // task_hierarchy_designer("taxi:manual", mdp); create_directory("output/" + mdp.name() + "/hierarchies"); print_hierarchy("output/" + mdp.name() + "/hierarchies/hierarchy_" + to_string(run_index)); } // Initialize the agent's innards void HH::initialize (const MDP& mdp) { HierLearner::initialize(mdp); avg_reward.resize(mdp.state().num_agents(), 0.0); avg_time.resize(mdp.state().num_agents(), 0.0); } // Update the agent's innards void HH::update (const State& next_state, const valarray& reward, const valarray& elapsed_time) { for (unsigned agent = 0; agent < next_state.num_agents(); ++agent) if (next_state.action_complete(agent) && !task_stack[agent].empty()) { HH_PrimTask* hh_primtask = dynamic_cast(task_stack[agent].front().task); if (!hh_primtask) throw HierException(__FILE__, __LINE__, "Primitive task expected."); // Updating the primitive task and popping it off the task stack hh_primtask->update(agent, *task_stack[agent].front().state, reward[agent], elapsed_time[agent]); task_stack[agent].pop_front(); // Updating the primitive task's parent dynamic_cast(task_stack[agent].front().task)->update(agent, task_stack[agent].front().parameters, *task_stack[agent].front().state, task_stack[agent].front().subtask, next_state); // Surfing through the task stack for tasks terminated in 'next_state' auto task_itr = task_stack[agent].begin(); // Top-of-stack auto task_itr_root = (++task_stack[agent].crbegin()).base(); // Root task iterator address while (task_itr != task_itr_root) { if (dynamic_cast(task_itr->task)->terminated(agent, task_itr->parameters, next_state)) { // Removing this task and all its children from the stack task_stack[agent].erase(task_stack[agent].begin(), ++task_itr); // Updating the parent task dynamic_cast(task_itr->task)->update(agent, task_itr->parameters, *task_itr->state, task_itr->subtask, next_state); } else ++task_itr; } } } hierlearning/learner/hier/hh/hh_task.h0000664000175000017500000000137612205065452020165 0ustar nevilleneville /*********************************************************************** HIERARCHICAL H TASK Neville Mehta ************************************************************************/ #pragma once #include #include #include "../../../domain/mdp.h" #include "../generic/task.h" class HH_Task : public virtual Task { protected: valarray _H; // H(s) table (it doubles as the Total_reward(s) table) valarray _Time; // Time(s) table; public: HH_Task(const string& task_name, const string& parameters, const string& state_variables, const MDP& mdp); virtual void initialize(); double h (const int& s) const { return _H[s]; } double time (const int& s) const { return _Time[s]; } virtual ~HH_Task() {} }; hierlearning/learner/hier/hh/hh_composite_task.cc0000664000175000017500000001430112277302554022403 0ustar nevilleneville /*********************************************************************** HIERARCHICAL H COMPOSITE TASK Neville Mehta ************************************************************************/ #include #include #include "../../../lib/common.h" #include "hh_composite_task.h" HH_CompositeTask::HH_CompositeTask (const string& task_name, const string& parameters, const string& state_variables, const vector& subtasks, bool (*function)(const unsigned& agent, const vector& parameters, const State& state), const Expression& expression, const MDP& mdp) : Task(task_name, parameters, state_variables, mdp), CompositeTask(task_name, parameters, state_variables, subtasks, function, expression, mdp), HH_Task(task_name, parameters, state_variables, mdp) { // Create the N(s,a) and N(s,a,s') tables _Nsa.resize(_num_bindings * _num_states, _num_subtasks); _Nsas.resize(_num_bindings * _num_states, _num_subtasks); } void HH_CompositeTask::add_subtask (const Subtask& subtask, const MDP& mdp) { CompositeTask::add_subtask(subtask, mdp); _H.resize(_num_bindings * _num_states); _Time.resize(_num_bindings * _num_states); _Nsa.resize(_num_bindings * _num_states, _num_subtasks); _Nsas.resize(_num_bindings * _num_states, _num_subtasks); } bool HH_CompositeTask::update_parameters (const vector& parameter_size) { if (Task::update_parameters(parameter_size)) { _H.resize(_num_bindings * _num_states); _Time.resize(_num_bindings * _num_states); _Nsa.resize(_num_bindings * _num_states, _num_subtasks); _Nsas.resize(_num_bindings * _num_states, _num_subtasks); return true; } return false; } void HH_CompositeTask::initialize () { HH_Task::initialize(); _Nsa = 0; // Initializing the N(s,a) tables // Initializing the N(s,a,s') table for (unsigned s = 0; s < _num_bindings * _num_states; s++) for (unsigned t = 0; t < _num_subtasks; t++) _Nsas(s,t).clear(); } // Calculate E[Time(s')] double HH_CompositeTask::expected_time (const int& s, const int& t) const { double E_T = 0.0; unordered_map Nsas = _Nsas(s,t); for (const auto& ns : Nsas) E_T += (double(ns.second) / _Nsa(s,t)) * _Time[ns.first]; return E_T; } // Calculate E[H(s')] double HH_CompositeTask::expected_h (const int& s, const int& t) const { double E_H = 0.0; unordered_map Nsas = _Nsas(s,t); for (const auto& ns : Nsas) E_H += (double(ns.second) / _Nsa(s,t)) * _H[ns.first]; return E_H; } unsigned HH_CompositeTask::greedy_policy (const unsigned& agent, const vector& parameters, const State& state) { vector adm_tasks = admissible_subtasks(agent, parameters, state); if (adm_tasks.size() == 1) return adm_tasks[0]; else { int s = hash(agent, parameters, state); double rho = (*_avg_time)[agent] > 0.0 ? (*_avg_reward)[agent]/(*_avg_time)[agent] : 0.0; double max_val = -numeric_limits::max(), eqn; vector gtasks; for (auto t = adm_tasks.cbegin(); t != adm_tasks.cend(); ++t) { const int subtask_hash = _subtask[*t].link->hash(agent, subtask_bindings(agent, parameters, state, *t), state); eqn = (dynamic_cast(_subtask[*t].link)->h(subtask_hash) - rho * dynamic_cast(_subtask[*t].link)->time(subtask_hash)) // Average-adjusted reward for the child + (expected_h(s, *t) - rho * expected_time(s, *t)); // Average-adjusted reward for completing the parent task if (max_val < eqn) { max_val = eqn; // Initiate the list of greedy subtasks gtasks.clear(); gtasks.push_back(*t); } else if (max_val == eqn) // Add to the list of greedy subtasks gtasks.push_back(*t); } unsigned num_greedy_actions = gtasks.size(); return gtasks[num_greedy_actions > 1 ? rand_int(num_greedy_actions) : 0]; } } inline double HH_CompositeTask::time_update (const unsigned& agent, const vector& parameters, const State& state) { int best_subtask = greedy_policy(agent, parameters, state); int s = hash(agent, parameters, state); return dynamic_cast(_subtask[best_subtask].link)->time(_subtask[best_subtask].link->hash(agent, subtask_bindings(agent, parameters, state, best_subtask), state)) + expected_time(s, best_subtask); } inline double HH_CompositeTask::total_reward_update (const unsigned& agent, const vector& parameters, const State& state) { int best_subtask = greedy_policy(agent, parameters, state); int s = hash(agent, parameters, state); return dynamic_cast(_subtask[best_subtask].link)->h(_subtask[best_subtask].link->hash(agent, subtask_bindings(agent, parameters, state, best_subtask), state)) + expected_h(s, best_subtask); } void HH_CompositeTask::update (const unsigned& agent, const vector& parameters, const State& state, const unsigned& t, const State& next_state) { int s = hash(agent, parameters, state); // P(s'|s,a) update _Nsas(s,t)[hash(agent, parameters, next_state)]++; // N(s,a,s') update _Nsa(s,t)++; // N(s,a) update _Time[s] = time_update(agent, parameters, state); _H[s] = total_reward_update(agent, parameters, state); } string HH_CompositeTask::print (const State& state) const { ostringstream out; for (unsigned b = 0; b < _num_bindings; b++) { out << "\n\n******** " << _name; auto parameters_and_state = unhash(0, b * _num_states, state); if (!parameters_and_state.first.empty()) out << parameters_and_state.first; out << " ********\n"; out << "\nH(s):\n"; for (unsigned s = b * _num_states; s < (b + 1) * _num_states; s++) if (_H[s] != 0.0) out << "[" << unhash(0, s, state).second->print() << "] " << _H[s] << endl; out << "\nTime(s):\n"; for (unsigned s = b * _num_states; s < (b + 1) * _num_states; s++) if (_Time[s] != 0.0) out << "[" << unhash(0, s, state).second->print() << "] " << _Time[s] << endl; out << "\nP(s'|s,a):\n"; for (unsigned s = b * _num_states; s < (b + 1) * _num_states; s++) { parameters_and_state = unhash(0, s, state); for (unsigned t = 0; t < _num_subtasks; t++) { unordered_map Nsas = _Nsas(s,t); for (const auto& ns : Nsas) { out << "[" << parameters_and_state.second->print() << "], " << t << ", ["; out << unhash(0, ns.first, state).second->print() << "] " << double(ns.second) / _Nsa(s,t) << endl; } } } } return out.str(); } hierlearning/learner/hier/hh/hh_composite_task.h0000664000175000017500000000355112205065452022244 0ustar nevilleneville /*********************************************************************** HIERARCHICAL H COMPOSITE TASK Neville Mehta ************************************************************************/ #pragma once #include #include #include #include "../../../lib/matrix.h" #include "../generic/composite_task.h" #include "hh_task.h" class HH_CompositeTask : public CompositeTask, public HH_Task { protected: matrix _Nsa; // N(s,a) table matrix> _Nsas; // N(s,a,s') table static valarray* _avg_reward; static valarray* _avg_time; double expected_time(const int& s, const int& t) const; // P(s'|s,a).time(s') summation double expected_h(const int& s, const int& t) const; // P(s'|s,a).h(s') summation double time_update(const unsigned& agent, const vector& parameters, const State& state); double total_reward_update(const unsigned& agent, const vector& parameters, const State& state); public: HH_CompositeTask(const string& task_name, const string& parameters, const string& state_variables, const vector& subtasks, bool (*function)(const unsigned& agent, const vector& parameters, const State& state), const Expression& expression, const MDP& mdp); static void set_rho (valarray* avg_reward, valarray* avg_time) { _avg_reward = avg_reward; _avg_time = avg_time; } void add_subtask(const Subtask& subtask, const MDP& mdp); bool update_parameters(const vector& parameter_size); virtual void initialize(); unsigned greedy_policy(const unsigned& agent, const vector& parameters, const State& state); virtual void update(const unsigned& agent, const vector& parameters, const State& state, const unsigned& t, const State& next_state); string print(const State& state) const; virtual ~HH_CompositeTask() {} }; hierlearning/learner/hier/hh/hh_primitive_task.h0000664000175000017500000000126312205065452022250 0ustar nevilleneville /*********************************************************************** HIERARCHICAL H PRIMITIVE TASK Neville Mehta ************************************************************************/ #pragma once #include #include "../generic/primitive_task.h" #include "hh_task.h" class HH_PrimTask : public PrimTask, public HH_Task { protected: valarray _N; public: HH_PrimTask(const string& task_name, const string& state_variables, const int& action, const MDP& mdp); void initialize(); void update(const unsigned& agent, const State& state, const double& reward, const double& elapsed_time); string print(const State& state) const; ~HH_PrimTask() {} }; hierlearning/learner/hier/hh/hh_root_task.cc0000664000175000017500000000642312277300472021367 0ustar nevilleneville /*********************************************************************** HIERARCHICAL H ROOT TASK Neville Mehta ************************************************************************/ #include #include #include #include "hh_root_task.h" HH_RootTask::HH_RootTask (const string& task_name, const string& parameters, const string& state_variables, const vector& subtasks, bool (*func)(const unsigned& agent, const vector& parameters, const State& state), const Expression& expression, const MDP& mdp) : Task(task_name, parameters, state_variables, mdp), HH_CompositeTask(task_name, parameters, state_variables, subtasks, func, expression, mdp) { _greedy_subtask.resize(mdp.state().num_agents()); _alpha.resize(mdp.state().num_agents()); } void HH_RootTask::initialize () { HH_CompositeTask::initialize(); _alpha = 1.0; } double HH_RootTask::avg_adjusted_update (const unsigned& agent, const vector& parameters, const State& state) { vector adm_tasks = admissible_subtasks(agent, parameters, state); int s = hash(agent, parameters, state); double rho = (*_avg_time)[agent] > 0.0 ? (*_avg_reward)[agent]/(*_avg_time)[agent] : 0.0; double max_val = -numeric_limits::max(), eqn; for (auto t = adm_tasks.cbegin(); t != adm_tasks.cend(); ++t) { const int subtask_hash = _subtask[*t].link->hash(agent, subtask_bindings(agent, parameters, state, *t), state); eqn = dynamic_cast(_subtask[*t].link)->h(subtask_hash) - rho * dynamic_cast(_subtask[*t].link)->time(subtask_hash) + expected_h(s, *t); if (max_val < eqn) max_val = eqn; } return max_val; } unsigned HH_RootTask::greedy_policy (const unsigned& agent, const vector& parameters, const State& state) { _greedy_subtask[agent] = true; return HH_CompositeTask::greedy_policy(agent, parameters, state); } unsigned HH_RootTask::exploratory_policy (const unsigned& agent, const vector& parameters, const State& state) { vector adm_tasks = admissible_subtasks(agent, parameters, state); unsigned num_admissible_tasks = adm_tasks.size(); if (num_admissible_tasks == 1) { _greedy_subtask[agent] = true; return adm_tasks[0]; } if (rand_real() < composite_task_parameters::EPSILON) // Random { _greedy_subtask[agent] = false; return adm_tasks[rand_int(num_admissible_tasks)]; } else return greedy_policy(agent, parameters, state); } void HH_RootTask::update (const unsigned& agent, const vector& parameters, const State& state, const unsigned& t, const State& next_state) { int s = hash(agent, parameters, state); // P(s'|s,a) update _Nsas(s,t)[hash(agent, parameters, next_state)]++; // N(s,a,s') update _Nsa(s,t)++; // N(s,a) update if (_greedy_subtask[agent]) // Root's subtask has been greedily selected { int subtask_hash = _subtask[t].link->hash(agent, subtask_bindings(agent, parameters, state, t), state); (*_avg_reward)[agent] += _alpha[agent] * (dynamic_cast(_subtask[t].link)->h(subtask_hash) + _H[hash(agent, parameters, next_state)] - _H[s] - (*_avg_reward)[agent]); (*_avg_time)[agent] += _alpha[agent] * (dynamic_cast(_subtask[t].link)->time(subtask_hash) - (*_avg_time)[agent]); _alpha[agent] /= (_alpha[agent] + 1); } _H[s] = avg_adjusted_update(agent, parameters, state); } hierlearning/learner/hier/hh/hh.h0000664000175000017500000000110412213033302017114 0ustar nevilleneville /************************************************* HIERARCHICAL H LEARNER Neville Mehta **************************************************/ #pragma once #include #include "../generic/hier_learner.h" class HH : public HierLearner { valarray avg_reward, avg_time; public: HH(const string& learner_name, const MDP& mdp, const unique_ptr& hierarchy, const unsigned& run_index); void initialize(const MDP& mdp); void update(const State& next_state, const valarray& reward, const valarray& elapsed_time); ~HH () {} }; hierlearning/learner/hier/hh/module.make0000664000175000017500000000021612205065452020507 0ustar nevillenevilleFOLDER = learner/hier/hh OBJ = hh_task.o hh_root_task.o hh_primitive_task.o hh_composite_task.o hh.o FILES += $(addprefix $(FOLDER)/, $(OBJ)) hierlearning/learner/hier/hier.h0000664000175000017500000000005312205065452017063 0ustar nevilleneville #include "maxq/maxq.h" #include "hh/hh.h" hierlearning/learner/hier/maxq/0000775000175000017500000000000012304226346016734 5ustar nevillenevillehierlearning/learner/hier/maxq/maxq_primitive_task.h0000664000175000017500000000237512205065452023173 0ustar nevilleneville /********************************************************************************** MAXQ PRIMITIVE TASK Neville Mehta ***********************************************************************************/ #pragma once #include #include #include #include "../generic/primitive_task.h" #include "maxq_task.h" namespace maxq_primitive_task { const double TABULAR_ALPHA = 0.1; // Learning rate for tables const double LINEAR_ALPHA = 0.001; // Learning rate for linear function approximators } class MaxQ_PrimTask : public PrimTask, public MaxQ_Task { public: enum Value_function_type {Tabular, Linear}; protected: Value_function_type _v_type; valarray _V; // V(s) table valarray _weight; // Weights valarray feature_vector(const unsigned& agent, const State& state) const; public: MaxQ_PrimTask(const string& task_name, const string& state_variables, const int& action, const MDP& mdp, const Value_function_type v_type = Tabular); void initialize(); double V(const unsigned& agent, const vector& parameters, const State& state) const; void update(const unsigned& agent, const State& state, const double& reward); string print(const State& state) const; ~MaxQ_PrimTask() {} }; hierlearning/learner/hier/maxq/maxq_primitive_task.cc0000664000175000017500000000457312277302462023337 0ustar nevilleneville /********************************************************************************** MAXQ PRIMITIVE TASK Neville Mehta ***********************************************************************************/ #include #include "../../../lib/common.h" #include "maxq_primitive_task.h" MaxQ_PrimTask::MaxQ_PrimTask (const string& task_name, const string& state_variables, const int& action, const MDP& mdp, const Value_function_type v_type) : Task(task_name, "", state_variables, mdp), PrimTask(task_name, state_variables, action, mdp), MaxQ_Task(task_name, "", state_variables, mdp), _v_type(v_type) { if (_v_type == Tabular) _V.resize(_num_states); else if (_v_type == Linear) _weight.resize(_variables.size() + 1); } void MaxQ_PrimTask::initialize () { if (_v_type == Tabular) _V = 0.0; else if (_v_type == Linear) _weight = 0.0; } inline valarray MaxQ_PrimTask::feature_vector (const unsigned& agent, const State& state) const { valarray feature(_variables.size() + 1); // State variables unsigned f = 0; for (const auto& v : _variables) switch (v.type) { case 0: feature[f++] = state.variable(v.value); break; default: feature[f++] = state.variable(agent * state.num_agent_variables() + v.value); break; } feature[f] = 1.0; return feature; } double MaxQ_PrimTask::V (const unsigned& agent, const vector& parameters, const State& state) const { if (_v_type == Tabular) return _V[hash(agent, vector(), state)]; else if (_v_type == Linear) return (_weight * feature_vector(agent, state)).sum(); else return 0.0; } void MaxQ_PrimTask::update (const unsigned& agent, const State& state, const double& reward) { if (_v_type == Tabular) { int s = hash(agent, vector(), state); _V[s] += maxq_primitive_task::TABULAR_ALPHA * (reward - _V[s]); } else if (_v_type == Linear) _weight += maxq_primitive_task::LINEAR_ALPHA * (reward - V(agent, vector(), state)) * feature_vector(agent, state); } string MaxQ_PrimTask::print (const State& state) const { ostringstream out; out << "\n\n******** " << _name << " ********\n"; if (_v_type == Tabular) { out << "\nV(s):\n"; for (unsigned s = 0; s < _num_states; ++s) if (_V[s] != 0.0) out << "[" << unhash(0, s, state).second->print() << "] " << _V[s] << endl; } else if (_v_type == Linear) out << "\nWeights: " << _weight << endl; return out.str(); } hierlearning/learner/hier/maxq/maxq.h0000664000175000017500000000073312213033230020041 0ustar nevilleneville /************************************************* MAXQ LEARNER Neville Mehta **************************************************/ #pragma once #include "../generic/hier_learner.h" class MaxQ : public HierLearner { public: MaxQ(const string& learner_name, const MDP& mdp, const unique_ptr& hierarchy, const unsigned& run_index); void update(const State& next_state, const valarray& reward, const valarray& duration); ~MaxQ() {} }; hierlearning/learner/hier/maxq/maxq.cc0000664000175000017500000000537112277301076020222 0ustar nevilleneville /***************************************************************************************** MAXQ LEARNER Neville Mehta ******************************************************************************************/ #include #include #include #include "maxq_composite_task.h" #include "maxq_primitive_task.h" #include "maxq.h" MaxQ::MaxQ (const string& learner_name, const MDP& mdp, const unique_ptr& hierarchy, const unsigned& run_index) : HierLearner(learner_name, mdp) { if (hierarchy) hierarchy->task_hierarchy_builder(mdp, Task_list); else flat_hierarchy_designer(mdp); // task_hierarchy_designer("taxi:manual", mdp); create_directory("output/" + mdp.name() + "/hierarchies"); print_hierarchy("output/" + mdp.name() + "/hierarchies/hierarchy_" + to_string(run_index)); } // Update the agent's innards void MaxQ::update (const State& next_state, const valarray& reward, const valarray& duration) { for (unsigned agent = 0; agent < next_state.num_agents(); ++agent) if (next_state.action_complete(agent) && !task_stack[agent].empty()) { MaxQ_PrimTask* maxq_primtask = dynamic_cast(task_stack[agent].front().task); if (!maxq_primtask) throw HierException(__FILE__, __LINE__, "Primitive task expected."); // Updating the primitive task and popping it off the task stack maxq_primtask->update(agent, *task_stack[agent].front().state, reward[agent]); task_stack[agent].pop_front(); // Updating the primitive task's parent dynamic_cast(task_stack[agent].front().task)->update(agent, task_stack[agent].front().parameters, *task_stack[agent].front().state, task_stack[agent].front().subtask, next_state, duration[agent]); // Surfing through the task stack for tasks terminated in 'next_state' auto task_itr = task_stack[agent].begin(); // Top-of-stack auto task_itr_root = (++task_stack[agent].crbegin()).base(); // Root task iterator address while (task_itr != task_itr_root) { task_itr->time += duration[agent]; if (dynamic_cast(task_itr->task)->terminated(agent, task_itr->parameters, next_state)) { double child_duration = task_itr->time; // Sending the current task's duration up the stack task_stack[agent].erase(task_stack[agent].begin(), ++task_itr); // Removing this task and all its children from the stack // Updating the parent task dynamic_cast(task_itr->task)->update(agent, task_itr->parameters, *task_itr->state, task_itr->subtask, next_state, child_duration); } else ++task_itr; } } } hierlearning/learner/hier/maxq/maxq_task.h0000664000175000017500000000126212205065452021075 0ustar nevilleneville /********************************************************************************** MAXQ TASK FRAMEWORK Neville Mehta ***********************************************************************************/ #pragma once #include #include #include #include "../../../domain/mdp.h" #include "../generic/task.h" class MaxQ_Task : public virtual Task { public: MaxQ_Task(const string& task_name, const string& parameters, const string& state_variables, const MDP& mdp) : Task(task_name, parameters, state_variables, mdp) {} virtual double V(const unsigned& agent, const vector& parameters, const State& state) const = 0; virtual ~MaxQ_Task () {} }; hierlearning/learner/hier/maxq/maxq_composite_task.cc0000664000175000017500000001033512277302360023317 0ustar nevilleneville /********************************************************************************** MAXQ COMPOSITE TASK Neville Mehta ***********************************************************************************/ #include #include #include "../../../lib/common.h" #include "maxq_composite_task.h" MaxQ_CompositeTask::MaxQ_CompositeTask (const string& task_name, const string& parameters, const string& state_variables, const vector& subtasks, bool (*function)(const unsigned& agent, const vector& parameters, const State& state), const Expression& expression, const MDP& mdp) : Task(task_name, parameters, state_variables, mdp), CompositeTask(task_name, parameters, state_variables, subtasks, function, expression, mdp), MaxQ_Task(task_name, parameters, state_variables, mdp) { // Create the C(s,a) table _C.resize(_num_bindings * _num_states, _num_subtasks); } void MaxQ_CompositeTask::add_subtask (const Subtask& subtask, const MDP& mdp) { CompositeTask::add_subtask(subtask, mdp); _C.resize(_num_bindings * _num_states, _num_subtasks); } bool MaxQ_CompositeTask::update_parameters (const vector& parameter_size) { if (Task::update_parameters(parameter_size)) { _C.resize(_num_bindings * _num_states, _num_subtasks); return true; } return false; } void MaxQ_CompositeTask::initialize () { // Initializing the C(s,a) table _C = 0.0; } inline double MaxQ_CompositeTask::V (const unsigned& agent, const vector& parameters, const State& state) const { if (terminated(agent, parameters, state)) return 0.0; else { vector adm_tasks = admissible_subtasks(agent, parameters, state); int s = hash(agent, parameters, state); double max_val = -numeric_limits::max(); for (auto t = adm_tasks.cbegin(); t != adm_tasks.cend(); ++t) { double eqn = dynamic_cast(_subtask[*t].link)->V(agent, subtask_bindings(agent, parameters, state, *t), state) + _C(s, *t); if (max_val < eqn) max_val = eqn; } return max_val; } } unsigned MaxQ_CompositeTask::greedy_policy (const unsigned& agent, const vector& parameters, const State& state) { vector adm_tasks = admissible_subtasks(agent, parameters, state); if (adm_tasks.size() == 1) return adm_tasks[0]; else { double max_val = -numeric_limits::max(); int s = hash(agent, parameters, state); vector gtasks; for (auto t = adm_tasks.cbegin(); t != adm_tasks.cend(); ++t) { double eqn = dynamic_cast(_subtask[*t].link)->V(agent, subtask_bindings(agent, parameters, state, *t), state) + _C(s, *t); if (max_val < eqn) { max_val = eqn; // Initiate the list of greedy subtasks gtasks.clear(); gtasks.push_back(*t); } else if (max_val == eqn) // Add to the list of greedy subtasks gtasks.push_back(*t); } unsigned num_greedy_actions = gtasks.size(); return gtasks[num_greedy_actions > 1 ? rand_int(num_greedy_actions) : 0]; } } void MaxQ_CompositeTask::update (const unsigned& agent, const vector& parameters, const State& state, const unsigned& t, const State& next_state, const double& N) { int s = hash(agent, parameters, state); _C(s, t) += maxq_composite_task::ALPHA * (pow(maxq_composite_task::GAMMA, N) * V(agent, parameters, next_state) - _C(s, t)); // Regular MaxQ-0 update } string MaxQ_CompositeTask::print (const State& state) const { ostringstream out; for (unsigned b = 0; b < _num_bindings; ++b) { out << "\n\n******** " << _name; auto parameters_and_state = unhash(0, b * _num_states, state); if (!parameters_and_state.first.empty()) out << parameters_and_state.first; out << " ********\n"; out << "\nC(s,a):\n"; for (unsigned s = b * _num_states; s < (b + 1) * _num_states; ++s) for (unsigned t = 0; t < _num_subtasks; ++t) if (_C(s, t) != 0.0) { parameters_and_state = unhash(0, s, state); out << "[" << parameters_and_state.second->print() << "], " << _subtask[t].link->name(); if (!_subtask[t].link->primitive()) { try { out << subtask_bindings(0, parameters_and_state.first, *parameters_and_state.second, t); } catch (const ParameterException&) { out << "(" << binding_str(t) << ")"; } } out << " " << _C(s, t) << endl; } } return out.str(); } hierlearning/learner/hier/maxq/maxq_composite_task.h0000664000175000017500000000271012205065452023156 0ustar nevilleneville /********************************************************************************** MAXQ COMPOSITE TASK Neville Mehta ***********************************************************************************/ #pragma once #include #include #include "../../../lib/matrix.h" #include "../../../hiergen/expression.h" #include "../generic/composite_task.h" #include "maxq_task.h" namespace maxq_composite_task { const double GAMMA = 1.0; // Discount factor const double ALPHA = 0.1; } class MaxQ_CompositeTask : public CompositeTask, public MaxQ_Task { protected: matrix _C; // Completion function public: MaxQ_CompositeTask(const string& task_name, const string& parameters, const string& state_variables, const vector& subtasks, bool (*function)(const unsigned& agent, const vector& parameters, const State& state), const Expression& expression, const MDP& mdp); void add_subtask(const Subtask& subtask, const MDP& mdp); bool update_parameters(const vector& parameter_size); void initialize(); double V(const unsigned& agent, const vector& parameters, const State& state) const; unsigned greedy_policy(const unsigned& agent, const vector& parameters, const State& state); void update(const unsigned& agent, const vector& parameters, const State& state, const unsigned& t, const State& next_state, const double& N); string print(const State& state) const; ~MaxQ_CompositeTask() {} }; hierlearning/learner/hier/maxq/module.make0000664000175000017500000000017512205065452021062 0ustar nevillenevilleFOLDER = learner/hier/maxq OBJ = maxq_primitive_task.o maxq_composite_task.o maxq.o FILES += $(addprefix $(FOLDER)/, $(OBJ)) hierlearning/learner/learner.h0000664000175000017500000000150212206263770016641 0ustar nevilleneville /************************************************* GENERIC LEARNER Neville Mehta **************************************************/ #pragma once #include "../domain/mdp.h" class Learner { protected: string _name; public: Learner (const string& learner_name) : _name(learner_name) {} string name () const { return _name; } virtual void initialize(const MDP& mdp) = 0; virtual void reset() = 0; virtual vector greedy_policy(const State& state) = 0; virtual vector exploratory_policy(const State& state) = 0; virtual vector learned_policy(const State& state) = 0; virtual void update(const State& next_state, const valarray& reward, const valarray& duration) = 0; virtual void write_text_file(const string& filename, const MDP& mdp) const = 0; virtual ~Learner () {} }; hierlearning/learner/flat/0000775000175000017500000000000012205065452015764 5ustar nevillenevillehierlearning/learner/flat/generic/0000775000175000017500000000000012304226346017401 5ustar nevillenevillehierlearning/learner/flat/generic/function.h0000664000175000017500000000224712206264166021407 0ustar nevilleneville /********************************************************************************** FUNCTION HEADER Neville Mehta ***********************************************************************************/ #pragma once #include #include "../../../domain/mdp.h" namespace function_parameters { const double EPSILON = 0.1; // Exploration factor } class Function { typedef pair Variable; list variables; protected: unsigned _num_states; unsigned _num_actions; public: Function(const string& state_variables, const unsigned& num_actions, const MDP& mdp); int hash (const unsigned& agent, const State& state) const; unique_ptr unhash(const unsigned& agent, int hash, const State& state) const; virtual void initialize() = 0; virtual unsigned greedy_policy(const unsigned& agent, const State& state) = 0; virtual unsigned exploratory_policy(const unsigned& agent, const State& state); virtual void update(const unsigned& agent, const State& state, const unsigned& action, const double& reward, const double& duration, const State& next_state) = 0; virtual string print(const State& state) const = 0; virtual ~Function () {} }; hierlearning/learner/flat/generic/function.cc0000664000175000017500000000401112205065452021530 0ustar nevilleneville /********************************************************************************** FUNCTION Neville Mehta ***********************************************************************************/ #include "../../../lib/common.h" #include "function.h" Function::Function (const string& state_variables, const unsigned& num_actions, const MDP& mdp) : _num_actions(num_actions) { vector state_variable_tokens = tokenize(state_variables, ", "); for (const auto& state_variable_token : state_variable_tokens) { if (state_variable_token.substr(0, 6) == "agent_") // Agent-based variables.emplace_back(1, mdp.state().variable_index(state_variable_token)); else variables.emplace_back(0, mdp.state().variable_index(state_variable_token)); } // Number of states _num_states = 1; for (const auto& v : variables) _num_states *= mdp.state().variable_size(v.second); } int Function::hash (const unsigned& agent, const State& state) const { int hash = 0; // State variables for (auto v = variables.crbegin(); v != variables.crend(); ++v) { int var; switch (v->first) { case 0: // Regular var = v->second; break; case 1: // Agent-based var = agent * state.num_agent_variables() + v->second; break; } hash = state.variable_size(var) * hash + state.variable(var); } return hash; } unique_ptr Function::unhash (const unsigned& agent, int hash, const State& state) const { unique_ptr _state(state.copy()); // Unhash state variables for (const auto& v : variables) { int var; switch (v.first) { case 0: // Regular var = v.second; break; case 1: // Agent-based var = agent * state.num_agent_variables() + v.second; break; } _state->variable(var) = hash % state.variable_size(var); hash /= state.variable_size(var); } return _state; } unsigned Function::exploratory_policy (const unsigned& agent, const State& state) { if (rand_real() < function_parameters::EPSILON) return rand_int(_num_actions); else return greedy_policy(agent, state); } hierlearning/learner/flat/generic/flat_learner.cc0000664000175000017500000000464612216517270022361 0ustar nevilleneville /***************************************************************************************** FLAT LEARNER Neville Mehta ******************************************************************************************/ #include #include "../../../lib/common.h" #include "flat_learner.h" string FlatLearner::mdp_state_variables (const MDP& mdp) const { vector state_variables = mdp.state().variables(); string state_variables_str; for (auto v = state_variables.cbegin(); v != state_variables.cend(); ++v) { if (v != state_variables.begin()) state_variables_str += ", "; state_variables_str += mdp.state().variable_name(*v); } return state_variables_str; } void FlatLearner::initialize (const MDP& mdp) { current_action.clear(); current_action.resize(mdp.state().num_agents(), 0); function->initialize(); } vector FlatLearner::greedy_policy (const State& state) { current_state = state.clone(); vector agent_actions(state.num_agents()); for (unsigned agent = 0; agent < state.num_agents(); ++agent) if (state.action_complete(agent)) { current_action[agent] = function->greedy_policy(agent, state); agent_actions[agent] = action_encoding[current_action[agent]]; } return agent_actions; } vector FlatLearner::exploratory_policy (const State& state) { current_state = state.clone(); vector agent_actions(state.num_agents()); for (unsigned agent = 0; agent < state.num_agents(); ++agent) if (state.action_complete(agent)) { current_action[agent] = function->exploratory_policy(agent, state); agent_actions[agent] = action_encoding[current_action[agent]]; } return agent_actions; } void FlatLearner::update (const State& next_state, const valarray& reward, const valarray& duration) { for (unsigned agent = 0; agent < next_state.num_agents(); ++agent) if (next_state.action_complete(agent)) function->update(agent, *current_state, current_action[agent], reward[agent], duration[agent], next_state); } // Write the agent's task values to a file void FlatLearner::write_text_file (const string& filename, const MDP& mdp) const { ofstream output(filename.c_str()); if (!output.is_open()) throw HierException(__FILE__, __LINE__, "Unable to write to " + filename); output << "Learner: " << _name << endl; output << "MDP: " << mdp.name() << endl; output << "Epsilon: " << function_parameters::EPSILON << endl; output << function->print(mdp.state()); output.close(); } hierlearning/learner/flat/generic/flat_learner.h0000664000175000017500000000244312303743206022211 0ustar nevilleneville /************************************************* FLAT LEARNER Neville Mehta **************************************************/ #pragma once #include #include #include #include "../../../domain/mdp.h" #include "../../learner.h" #include "function.h" class FlatLearner : public Learner { const vector action_encoding; // Action encodings unique_ptr current_state; // Current states of the agents vector current_action; // Current actions of the agents unique_ptr function; protected: string mdp_state_variables(const MDP& mdp) const; public: FlatLearner (const string& learner_name, unique_ptr function, const MDP& mdp) : Learner(learner_name), action_encoding(mdp.actions()), current_action(vector(mdp.state().num_agents(), 0)), function(move(function)) {} void initialize(const MDP& mdp); void reset () {} vector greedy_policy(const State& state); vector exploratory_policy(const State& state); vector learned_policy (const State& state) { return greedy_policy(state); } void update(const State& next_state, const valarray& reward, const valarray& duration); void write_text_file(const string& filename, const MDP& mdp) const; virtual ~FlatLearner () {} }; hierlearning/learner/flat/generic/module.make0000664000175000017500000000014712205065452021526 0ustar nevillenevilleFOLDER = learner/flat/generic OBJ = function.o flat_learner.o FILES += $(addprefix $(FOLDER)/, $(OBJ)) hierlearning/learner/flat/q/0000775000175000017500000000000012304226346016225 5ustar nevillenevillehierlearning/learner/flat/q/q_function.cc0000664000175000017500000000370412206472002020676 0ustar nevilleneville /********************************************************************************** Q FUNCTION Neville Mehta ***********************************************************************************/ #include #include #include "../../../lib/common.h" #include "q_function.h" Q_function::Q_function (const string& state_variables, const unsigned& num_actions, const MDP& mdp) : Function(state_variables, num_actions, mdp) { resize(_num_states, num_actions); } double Q_function::max_Q (const int& s) const { double max_val = -numeric_limits::max(); for (unsigned a = 0; a < _num_actions; ++a) if (max_val < (*this)(s,a)) max_val = (*this)(s,a); return max_val; } unsigned Q_function::greedy_policy (const unsigned& agent, const State& state) { int s = hash(agent, state); double max_val = -numeric_limits::max(); unsigned gtask = 0; unsigned ties = 0; for (unsigned a = 0; a < _num_actions; ++a) { if (max_val < (*this)(s,a)) { max_val = (*this)(s,a); gtask = a; ties = 1; } else if (max_val == (*this)(s,a) && rand_real() < 1.0/++ties) gtask = a; } return gtask; } void Q_function::update (const unsigned& agent, const State& state, const unsigned& action, const double& reward, const double& duration, const State& next_state) { int s = hash(agent, state); (*this)(s, action) += q_function_parameters::ALPHA * (reward + pow(q_function_parameters::GAMMA, duration) * max_Q(hash(agent, next_state)) - (*this)(s, action)); } string Q_function::print (const State& state) const { ostringstream out; out << "Alpha: " << q_function_parameters::ALPHA << endl; out << "\nQ(s,a):\n"; for (unsigned s = 0; s < _num_states; ++s) { unsigned a; for (a = 0; a < _num_actions; ++a) if ((*this)(s,a) != 0.0) break; if (a < _num_actions) { out << "[" << unhash(0, s, state)->print() << "]"; for (a = 0; a < _num_actions; ++a) out << " " << (*this)(s,a); out << endl; } } return out.str(); } hierlearning/learner/flat/q/q_function.h0000664000175000017500000000167712276076746020575 0ustar nevilleneville /********************************************************************************** Q FUNCTION HEADER Neville Mehta ***********************************************************************************/ #pragma once #include "../../../lib/matrix.h" #include "../generic/function.h" namespace q_function_parameters { const double ALPHA = 0.1; // Learning rate const double GAMMA = 1.0; // Discount factor } class Q_function : public Function, public matrix { double max_Q(const int& s) const; // max_a Q(s,a) public: Q_function(const string& state_variables, const unsigned& num_actions, const MDP& mdp); void initialize () { *_data = 0.0; } unsigned greedy_policy(const unsigned& agent, const State& state); void update(const unsigned& agent, const State& state, const unsigned& action, const double& reward, const double& duration, const State& next_state); string print(const State& state) const; ~Q_function () {} }; hierlearning/learner/flat/q/module.make0000664000175000017500000000012412205065452020345 0ustar nevillenevilleFOLDER = learner/flat/q OBJ = q_function.o FILES += $(addprefix $(FOLDER)/, $(OBJ)) hierlearning/learner/flat/q/q.h0000664000175000017500000000067112206455470016645 0ustar nevilleneville /************************************************* Q LEARNER Neville Mehta **************************************************/ #pragma once #include "../generic/flat_learner.h" #include "q_function.h" class Q : public FlatLearner { public: Q (const string& learner_name, const MDP& mdp) : FlatLearner(learner_name, unique_ptr(new Q_function(mdp_state_variables(mdp), mdp.actions().size(), mdp)), mdp) {} ~Q () {} }; hierlearning/learner/flat/flat.h0000664000175000017500000000004312205065452017060 0ustar nevilleneville #include "q/q.h" #include "h/h.h" hierlearning/learner/flat/h/0000775000175000017500000000000012304226346016214 5ustar nevillenevillehierlearning/learner/flat/h/h_function.h0000664000175000017500000000251012205065452020516 0ustar nevilleneville /********************************************************************************** H FUNCTION HEADER Neville Mehta ***********************************************************************************/ #pragma once #include #include #include "../../../lib/matrix.h" #include "../generic/function.h" class H_function : public Function { valarray _H; matrix _Reward; matrix _Time; matrix _Nsa; matrix> _Nsas; valarray _avg_reward; valarray _avg_time; valarray _alpha; valarray _greedy_action_selected; double rho (const unsigned& agent) const { return _avg_time[agent] > 0.0 ? _avg_reward[agent] / _avg_time[agent] : 0.0; } double expected_H(const int& s, const int& a) const; double max_H(const unsigned& agent, const State& state) const; public: H_function(const string& state_variables, const unsigned& num_actions, const MDP& mdp); void initialize(); unsigned greedy_policy(const unsigned& agent, const State& state); unsigned exploratory_policy(const unsigned& agent, const State& state); void update(const unsigned& agent, const State& state, const unsigned& action, const double& reward, const double& duration, const State& next_state); string print(const State& state) const; ~H_function () {} }; hierlearning/learner/flat/h/h.h0000664000175000017500000000067112206455512016620 0ustar nevilleneville /************************************************* H LEARNER Neville Mehta **************************************************/ #pragma once #include "../generic/flat_learner.h" #include "h_function.h" class H : public FlatLearner { public: H (const string& learner_name, const MDP& mdp) : FlatLearner(learner_name, unique_ptr(new H_function(mdp_state_variables(mdp), mdp.actions().size(), mdp)), mdp) {} ~H () {} }; hierlearning/learner/flat/h/h_function.cc0000664000175000017500000001130712303745444020665 0ustar nevilleneville /********************************************************************************** H FUNCTION Neville Mehta ***********************************************************************************/ #include #include #include "../../../lib/common.h" #include "h_function.h" H_function::H_function (const string& state_variables, const unsigned& num_actions, const MDP& mdp) : Function(state_variables, num_actions, mdp) { _H.resize(_num_states); _Reward.resize(_num_states, _num_actions); _Time.resize(_num_states, _num_actions); _Nsa.resize(_num_states, _num_actions); _Nsas.resize(_num_states, _num_actions); _avg_reward.resize(mdp.state().num_agents()); _avg_time.resize(mdp.state().num_agents()); _alpha.resize(mdp.state().num_agents()); _greedy_action_selected.resize(mdp.state().num_agents()); } void H_function::initialize () { _H = 0.0; _Reward = 0.0; _Time = 0.0; _Nsa = 0; // Initializing the N(s,a,s') table for (unsigned s = 0; s < _num_states; s++) for (unsigned a = 0; a < _num_actions; a++) _Nsas(s,a).clear(); _avg_reward = 0.0; _avg_time = 0.0; _alpha = 1.0; } // Calculate E[H(s')] double H_function::expected_H (const int& s, const int& a) const { double E_H = 0.0; unordered_map Nsas = _Nsas(s,a); for (const auto& ns : Nsas) E_H += (double(ns.second) / _Nsa(s,a)) * _H[ns.first]; return E_H; } double H_function::max_H (const unsigned& agent, const State& state) const { int s = hash(agent, state); double max_val = -numeric_limits::max(); for (unsigned a = 0; a < _num_actions; ++a) { double eqn = _Reward(s,a) - rho(agent) * _Time(s,a) + expected_H(s,a); if (max_val < eqn) max_val = eqn; } return max_val; } unsigned H_function::greedy_policy (const unsigned& agent, const State& state) { int s = hash(agent, state); _greedy_action_selected[agent] = true; double max_val = -numeric_limits::max(); unsigned gtask = 0; unsigned ties = 0; for (unsigned a = 0; a < _num_actions; ++a) { double eqn = _Reward(s,a) - rho(agent) * _Time(s,a) + expected_H(s,a); if (max_val < eqn) { max_val = eqn; gtask = a; ties = 1; } else if (max_val == eqn && rand_real() < 1.0/++ties) gtask = a; } return gtask; } unsigned H_function::exploratory_policy (const unsigned& agent, const State& state) { if (rand_real() < function_parameters::EPSILON) { _greedy_action_selected[agent] = false; return rand_int(_num_actions); } else { _greedy_action_selected[agent] = true; return greedy_policy(agent, state); } } void H_function::update (const unsigned& agent, const State& state, const unsigned& action, const double& reward, const double& duration, const State& next_state) { int s = hash(agent, state); // P(s'|s,a) update _Nsas(s, action)[hash(agent, next_state)]++; _Nsa(s, action)++; _Time(s, action) += (duration - _Time(s, action)) / _Nsa(s, action); _Reward(s, action) += (reward - _Reward(s, action)) / _Nsa(s, action); // rho update if (_greedy_action_selected[agent]) { _avg_reward[agent] += _alpha[agent] * (_Reward(s, action) + _H[hash(agent, next_state)] - _H[s] - _avg_reward[agent]); _avg_time[agent] += _alpha[agent] * (_Time(s, action) - _avg_time[agent]); _alpha[agent] /= (_alpha[agent] + 1); } _H[s] = max_H(agent, state); } string H_function::print (const State& state) const { ostringstream out; out << "\nRho = " << valarray(_avg_reward / _avg_time); out << "\nAverage reward = " << _avg_reward; out << "\nAverage time = " << _avg_time << endl; out << "\nH(s):\n"; for (unsigned s = 0; s < _num_states; ++s) if (_H[s] != 0.0) out << "[" << unhash(0, s, state)->print() << "] " << _H[s] << endl; out << "\nReward(s,a):\n"; for (unsigned s = 0; s < _num_states; ++s) { unsigned a; for (a = 0; a < _num_actions; ++a) if (_Reward(s,a) != 0.0) break; if (a < _num_actions) { out << "[" << unhash(0, s, state)->print() << "]"; for (a = 0; a < _num_actions; ++a) out << " " << _Reward(s,a); out << endl; } } out << "\nTime(s,a):\n"; for (unsigned s = 0; s < _num_states; ++s) { unsigned a; for (a = 0; a < _num_actions; ++a) if (_Time(s,a) != 0.0) break; if (a < _num_actions) { out << "[" << unhash(0, s, state)->print() << "]"; for (a = 0; a < _num_actions; ++a) out << " " << _Time(s,a); out << endl; } } out << "\nP(s'|s,a):\n"; for (unsigned s = 0; s < _num_states; ++s) { unique_ptr _state = unhash(0, s, state); for (unsigned a = 0; a < _num_actions; a++) { unordered_map Nsas = _Nsas(s,a); for (const auto& ns : Nsas) { out << "[" << _state->print() << "], " << a << ", ["; out << unhash(0, ns.first, state)->print() << "] " << double(ns.second) / _Nsa(s,a) << endl; } } } return out.str(); } hierlearning/learner/flat/h/module.make0000664000175000017500000000012412205065452020334 0ustar nevillenevilleFOLDER = learner/flat/h OBJ = h_function.o FILES += $(addprefix $(FOLDER)/, $(OBJ)) hierlearning/learner/learners.h0000664000175000017500000000005712205065452017024 0ustar nevilleneville #include "flat/flat.h" #include "hier/hier.h"