Merge pull request #37 from JuliaRobotics/feat/3Q20/injdelay

add injectDelayBefore and watchdog

Author: dehann

Project.toml

@@ -2,7 +2,7 @@ name = "FunctionalStateMachine"
 uuid = "3e9e306e-7e3c-11e9-12d2-8f8f67a2f951"
 keywords = ["state machine"]
 desc = "Functional state machine with stepping and visualization tools."
-version = "0.2.7"
+version = "0.2.8"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

README.md

@@ -55,6 +55,26 @@ statemachine = StateMachine{Nothing}(next=foo!)
 while statemachine(nothing, iterlimit=1); end
 ```
 
+### Watchdog Timeout
+
+Sometimes it is useful to know that an FSM process will exit, either as intended or by throwing an error on timeout (much like a [Watchdog Timer](https://en.wikipedia.org/wiki/Watchdog_timer)).  FSM uses Base.`InterruptException()` as a method of stopping a task that expires a `timeout::Real` [seconds].  Note, this functionality is not included by default in order to preserve a small memory footprint.  To use the timeout feature simply call the state machine with a timeout duration:
+```julia
+userdata = nothing # any user data of type T
+timeout = 3.0
+while statemachine(userdata, timeout, verbose=true); end
+```
+
+### Recording Verbose Output to File
+
+Experience has shown that when a state machine gets stuck, it is often useful to write the `verbose` steps out to file as a bare minimum guide of where a system might be failing.  This can be done by passing in a `::IOStream` handle into `verbosefid`:
+```julia
+fid = open("/tmp/verboseFSM_001.log","w")
+while statemachine(userdata, verbose=true, verbosefid=fid); end
+close(fid)
+```
+
+This particular structure is choosen so that `@async` or other multithreaded uses of FSM can still write to a common `fid` and also allow the user to `flush(fid)` and `close(fid)` regardless of whether the FSM has stalled.  Might seem "boilerplate-esque", but it's much easier for developers to snuff out bugs in highly complicted interdependent and multithreaded, multi-state-machine architectures.
+
 ## With User Data and History
 
 ```julia

src/StateMachine.jl

@@ -25,11 +25,25 @@ mutable struct StateMachine{T}
   StateMachine{T}(;next=emptyState, iter::Int=0, name::AbstractString="") where T = new{T}(next, iter, Vector{Tuple{DateTime, Int, Function, T}}(), name)
 end
 
+
+
 """
     $SIGNATURES
 
 Run state machine function (as functor).
 
+Notes
+- `timeout::Union{Real,Nothing}` is optional with default `=nothing`.
+  - this code is skipped in lowered and llvm code if not used
+  - subroutine will use `pollinterval::Real` [seconds] to interrogate during `timeout::Real` [seconds] period.
+- can stop FSM early by using any of the following:
+  - `breakafter`, `iterlimit`.
+- Can `injectDelayBefore` a function `st.next` to help with debugging.
+- can print FSM steps with `verbose=true`.
+  - `verbosefid::IOStream` is used as destination for verbose output, default is `stdout`.
+- FSM steps and `userdata` can be recorded in standard `history` format using `recordhistory=true`.
+- `housekeeping_cb` is callback to give user access to `StateMachine` internals and opportunity to insert bespoke operations.
+
 Example
 ```julia
 bar!(usrdata) = IncrementalInference.exitStateMachine
@@ -40,23 +54,45 @@ usrdata = nothing
 while st(usrdata); end
 ```
 """
-function (st::StateMachine{T})( userdata::T=nothing;
+function (st::StateMachine{T})( userdata::T=nothing,
+                                timeout::Union{Nothing,<:Real}=nothing;
+                                pollinterval::Real=0.05,
                                 breakafter::Function=exitStateMachine,
                                 verbose::Bool=false,
+                                verbosefid=stdout,
                                 iterlimit::Int=-1,
+                                injectDelayBefore::Union{Nothing,Pair{<:Function, <:Real}}=nothing,
                                 recordhistory::Bool=false,
                                 housekeeping_cb::Function=(st)->()  ) where {T}
   #
   st.iter += 1
   # verbose print to help debugging
-  !verbose ? nothing : println("FSM $(st.name), iter=$(st.iter) -- $(st.next)")
+  !verbose ? nothing : println(verbosefid, "FSM $(st.name), iter=$(st.iter) -- $(st.next)")
   # early exit plumbing
   retval = st.next != breakafter && (iterlimit == -1 || st.iter < iterlimit)
   # record steps for later
-  recordhistory ? push!(st.history, (Dates.now(), st.iter, deepcopy(st.next), deepcopy(userdata))) : nothing
+  T0 = Dates.now()
+  recordhistory ? push!(st.history, (T0, st.iter, deepcopy(st.next), deepcopy(userdata))) : nothing
   # user has some special situation going on.
   housekeeping_cb(st)
-  st.next = st.next(userdata)
+  (injectDelayBefore !== nothing && injectDelayBefore[1] == st.next) ? sleep(injectDelayBefore[2]) : nothing
+  if timeout === nothing
+    # no watchdog, just go and optimize llvm lowered code
+    st.next = st.next(userdata)
+  else
+    # add the watchdog into the llvm lowered code
+    currtsk = current_task()
+    # small amount of memory usage, but must guarantee InterruptException is not accidently fired during next step.
+    doneWatchdog = Base.RefValue{Int}(0) 
+    wdt = @async begin
+      # wait for watchdog timeperiod in a seperate co-routine
+      res = timedwait(()->doneWatchdog[]==1, timeout, pollint=pollinterval)
+      # Two requirements needed to interrupt FSM step
+      res == :timed_out && doneWatchdog[] == 0 ? schedule(currtsk, InterruptException(), error=true) : nothing
+    end
+    st.next = st.next(userdata)
+    doneWatchdog[] = 1
+  end
   return retval
 end
 

test/testStateMachine.jl

@@ -52,6 +52,29 @@ while statemachine(nothing, verbose=true); end
 end
 
 
+@testset "test watchdog timeout" begin
+
+function longwait(x)
+  @info "starting stagnant function call, but should not see it's end (but watchdog timeout)"
+  while true 
+    print(".")
+    sleep(0.5)
+  end
+  @info "done with longwait"
+  return exitStateMachine
+end
+
+statemachine = StateMachine{Nothing}(next=longwait)
+try
+  while statemachine(nothing, 2.0, verbose=true); end
+catch e
+  @info " watchdog test, successfully caught exception for stagnant FSM step"
+  @test_throws InterruptException throw(e)
+end
+
+end
+
+
 @testset "test recording and rendering of an FSM run" begin
 
 statemachine = StateMachine{Nothing}(next=foo!)