A Dynamic MDP (DMDP) is used when rewards depend on the transition (the action taken from a state) rather than just the state itself.
- Extend the
DMDP class. - Implement
R(self, state, action) to return self.rewards[state][action]. - Implement
T(self, state, action) to return self.transitions[state][action]. - Because the standard
value_iteration assumes state-only rewards, you must use a modified version: value_iteration_dmdp(dmdp, epsilon). - Use a custom
best_policy_dmdp(dmdp, U) function to derive the policy from utilities U.
# Custom DMDP implementation snippet
class CustomDMDP(DMDP):
def T(self, state, action):
if action is None:
return [(0.0, state)]
else:
return [(prob, new_state) for new_state, prob in self.t[state][action].items()]
def R(self, state, action):
if action is None:
return 0
else:
return self.rewards[state][action]
# Solving with custom value iteration
def value_iteration_dmdp(dmdp, epsilon=0.001):
U1 = {s: 0 for s in dmdp.states}
R, T, gamma = dmdp.R, dmdp.T, dmdp.gamma
while True:
U = U1.copy()
delta = 0
for s in dmdp.states:
U1[s] = max([(R(s, a) + gamma*sum([(p*U[s1]) for (p, s1) in T(s, a)])) for a in dmdp.actions(s)])
delta = max(delta, abs(U1[s] - U[s]))
if delta < epsilon * (1 - gamma) / gamma:
return U
