;;; eve-online.el -- EVE-Online Tools
;;
;;; Copyright (C) 2008 Kyle W T Sherman
;;
;; Author:   Kyle W T Sherman <kylewsherman at gmail dot com>
;; Created:  2008-07-21
;; Version:  1.0
;; Keywords: eve-online mmorpg
;;
;;; Commentary:
;;
;; TODO: write me
;;
;; Provides `marketdata-gateway-query' function to POST the current XML buffer
;; to an appropriate gateway and display the results in a new buffer.
;; Standard servers are listed in `marketdata-gateway-servers' and the current
;; one being used is stored in `marketdata-gateway-server'.
;;
;; The 'To', 'Created', and 'Expires' tags are automatically populated before
;; submission.
;;
;; You must be using Emacs 22 or later for this module to work.  (It uses
;; timing code not found in earlier versions.)
;;
;;; Installation:
;;
;; Put `marketdata-gateway.el' where you keep your elisp files and add
;; something like the following to your .emacs file:
;;
;;   (require 'marketdata-gateway)
;;
;; Also, make sure the `wget' and `date' commands are in your path.
;;
;; Can be customized with the following command:
;;
;;   (customize-group "marketdata-gateway")
;;
;;; Usage:
;;
;; Visit one of the XML files found here:
;; `~/svn/MarketData/dj/cylon/trunk/admin-tools/gateway-monitor/post' (for
;; example, the `check-marketdatagateway-post' file).  Then run
;; `marketdata-gateway-query' and either type in a server name (resolvable IP
;; address), use "all" for all production servers, or press enter for a list
;; you can pick from.  The XML request will be posted to the server and the
;; response displayed.
;;
;; You may also call `marketdata-gateway-query' directly via
;; \\[eval-expression] with the server name as a parameter.
;;
;; You may run a batch of tests with `marketdata-gateway-query-batch', which
;; will run each query `marketdata-gateway-batch-size' times.
;;
;; Examples:
;;
;;   (marketdata-gateway-query "localhost")
;;   (marketdata-gateway-query "localhost" 10)
;;   (marketdata-gateway-query-batch "localhost")

;;; Code:

;; cl
(require 'cl)

;; easymenu
(require 'easymenu)

;; widget
(require 'widget)

;; server list
(defcustom marketdata-gateway-servers
  '(("sbkmdpwebapi" . "sbkmdpwebapi.mdprod.dowjones.net")
    ("secmdpwebapi" . "secmdpwebapi.mdprod.dowjones.net")
    ("sbkmdwebp01" . "sbkmdwebp01.mdprod.dowjones.net")
    ("sbkmdwebp02" . "sbkmdwebp02.mdprod.dowjones.net")
    ("sbkmdwebp03" . "sbkmdwebp03.mdprod.dowjones.net")
    ("sbkmdwebp04" . "sbkmdwebp04.mdprod.dowjones.net")
    ("secmdwebp01" . "secmdwebp01.mdprod.dowjones.net")
    ("secmdwebp02" . "secmdwebp02.mdprod.dowjones.net")
    ("secmdwebp03" . "secmdwebp03.mdprod.dowjones.net")
    ("secmdwebp04" . "secmdwebp04.mdprod.dowjones.net"))
  "List of MarketData Gateway servers that may be queried.
\nEach element is a tuple containing a name/server pair."
  :type 'list
  :group 'marketdata-gateway)

;; gateway info
(defcustom marketdata-gateway-info
  '(("http://service.marketwatch.com/ws/2006/08/dataDictionaryGateway" . ((:name . "DataDictionary") (:port . 9205)))
    ("http://service.marketwatch.com/ws/2006/07/marketDataGateway" . ((:name . "MarketData") (:port . 9206)))
    ("http://service.dowjones.com/ws/2006/11/symbologyGateway" . ((:name . "Symbology") (:port . 9207)))
    ("http://service.dowjones.com/ws/2007/09/quoteGateway" . ((:name . "Quote") (:port . 9208)))
    ("http://service.dowjones.com/ws/2007/10/timeseriesGateway" . ((:name . "TimeSeries") (:port . 9209))))
  "List of MarketData Gateway server query patterns and info about them.
\nEach element is a tuple containing a query-pattern/info pair.
The info is an association list of named data items as follows:
\n  :name - gateway name
  :port - gateway port"
  :type 'list
  :group 'marketdata-gateway)

;; request timeout
(defcustom marketdata-gateway-request-timeout
  30
  "*Number of seconds to wait for a query response."
  :type 'integer
  :group 'marketdata-gateway)

;; batch size
(defcustom marketdata-gateway-batch-size
  100
  "*Number of requests in a batch run."
  :type 'integer
  :group 'marketdata-gateway)

;; show output
(defcustom marketdata-gateway-show-output
  t
  "*Whether to show every response in a buffer or not (t or nil respectively)."
  :type 'boolean
  :group 'marketdata-gateway)

;; format output
(defcustom marketdata-gateway-format-output
  t
  "*Whether to reformat the output xml or not (t or nil respectively)."
  :type 'boolean
  :group 'marketdata-gateway)

;; initial server history
(defcustom marketdata-gateway-initial-query-history
  '("sbkmdpwebapi secmdpwebapi"
    "sbkmdwebp01 sbkmdwebp02 sbkmdwebp03 sbkmdwebp04 secmdwebp01 secmdwebp02 secmdwebp03 secmdwebp04")
  "*Initial list of server lists to query."
  :type 'list
  :group 'marketdata-gateway)

;; server history
(defvar marketdata-gateway-query-history
  ;; `(,@marketdata-gateway-initial-query-history
  ;;   ,(do ((servers marketdata-gateway-servers (cdr servers))
  ;;         (result ""))
  ;;        ((not servers) result)
  ;;      (if (cdr servers)
  ;;          (setq result (concat result (caar servers) " "))
  ;;        (setq result (concat result (caar servers))))))
  nil
  "History of `marketdata-gateway-query' query arguments.
\nInitialized with `marketdata-gateway-query-history' function.")

;; reset server history
(defun marketdata-gateway-query-history-reset ()
  "Reset `marketdata-gateway-query-history' to initial values."
  (setq marketdata-gateway-query-history
        (append marketdata-gateway-initial-query-history
                (do ((servers marketdata-gateway-servers (cdr servers))
                     (result ""))
                    ((not servers) (list result))
                  (if (cdr servers)
                      (setq result (concat result (caar servers) " "))
                    (setq result (concat result (caar servers))))))))

;; push to server history
(defun marketdata-gateway-query-history-push (servers)
  "Push SERVERS onto `marketdata-gateway-query-history' list
after removing any existing entries."
  ;; convert string server to list, if needed
  (when (stringp servers)
    (setq servers (split-string servers)))
  ;; continue if there are servers
  (when servers
    ;; flatten list to string
    (setq servers (join-strings-delimiter servers " "))
    ;; remove existing entry if needed
    (when (member servers marketdata-gateway-query-history)
      (setq marketdata-gateway-query-history (remove servers marketdata-gateway-query-history)))
    ;; add to history
    (push servers marketdata-gateway-query-history)))

;; initialize history
(marketdata-gateway-query-history-reset)

;; select server buffer name
(defvar marketdata-gateway-menu-buffer-name
  "*Marketdata-Gateway-Menu*"
  "Buffer name to use for menu.")

;; process status buffer name
(defvar marketdata-gateway-process-status-buffer-name
  "*MarketData-Gateway-Process-Status*"
  "Buffer name to use for showing process status.")

;; temp directory to store request files
(defvar marketdata-gateway-tmp-dir-request
  (expand-file-name "~/tmp/marketdata-gateway-request")
  "Temp directory to store request post files.")

;; temp directory to store response files
(defvar marketdata-gateway-tmp-dir-response
  (expand-file-name "~/tmp/marketdata-gateway-response")
  "Temp directory to store gateway responses.")

;; output buffer name
(defun marketdata-gateway-output-buffer-name (server)
  "Generate output buffer name."
  ;;(concat "*" server ":" (buffer-name) "*")
  (concat server ":" (buffer-name)))

;; output file name
(defun marketdata-gateway-output-file-name (buffer)
  "Generate an output file name from a BUFFER name."
  (concat marketdata-gateway-tmp-dir-response "/" buffer))

;; gateway info
(defun marketdata-gateway-info (&optional item)
  "Return requested gateway info for XML request based on the
SOAP header in the current buffer."
  (save-excursion
    (goto-char (point-min))
    (let (info)
      ;; loop through all gateways checking buffer for a match
      (dolist (gateway marketdata-gateway-info)
        (unless info
          (when (re-search-forward (car gateway) nil t)
            (setq info (cdr gateway)))))
      ;; either return all info or requested item
      (if item
          (cdr (assq item info))
        info))))

;; date offset
(defun marketdata-gateway-date-offset (&optional offset)
  "Return current UTC date plus OFFSET seconds."
  (unless offset
    (setq offset 0))
  (substring
   (shell-command-to-string
    (concat
     "TZ=UTC date -d \"" (shell-quote-argument (number-to-string offset))
     " sec\" \"+%Y-%m-%dT%H:%M:%SZ\""
     )) 0 -1))

;; prepare data
(defun marketdata-gateway-prepare-data (server port)
  "Prepare post data in current buffer for request."
  (let ((created (marketdata-gateway-date-offset 0))
        (expires (marketdata-gateway-date-offset 5)))
    (save-excursion
      ;; replace double quotes with single quotes in data
      ;; (goto-char (point-min))
      ;; (while (re-search-forward "\"" nil t)
      ;;   (replace-match "'"))
      ;; replace wsa:To fields with server:port
      (goto-char (point-min))
      (while (re-search-forward "<wsa:To>.*</wsa:To>" nil t)
        (replace-match (concat "<wsa:To>http://" server ":" port "</wsa:To>")))
      ;; replace wsu:Created fields with current timestamp
      (goto-char (point-min))
      (while (re-search-forward "#{created}" nil t)
        (replace-match created))
      ;; replace wsu:Expires fields with current timestamp
      (goto-char (point-min))
      (while (re-search-forward "#{expires}" nil t)
        (replace-match expires))
      )))

;; marketdata gateway menu
(defun marketdata-gateway-menu (&optional server batch)
  "Create a menu of options the user may select from.
\nIf SERVER is non-nil, it sets the default servers to query.
If BATCH is non-nil, it is passed on to `marketdata-gateway-query' calls."
  (interactive)
  ;; list to string function
  (defun list-to-string (lst)
    "Return space delimited version of items in LST."
    (do ((x lst (cdr x))
         result)
        ((not x) (apply 'concat (reverse result)))
      (if (cdr x)
          (push (concat (car x) " ") result)
        (push (car x) result))))
  (let (buffer                          ; menu buffer
        (post-buffer-name (buffer-name)) ; post buffer name
        query-history                   ; query history
        all-servers                     ; all servers ever queried
        last-history                    ; last history item
        last-history-servers)           ; last history server list
    ;; setup buffer
    (setq buffer (generate-new-buffer marketdata-gateway-menu-buffer-name))
    (set-buffer buffer)
    (kill-all-local-variables)
    ;; use local variable to store server list
    (make-local-variable 'widget-servers)
    ;; add server to top of history, if given
    (when server
      (marketdata-gateway-query-history-push server))
    ;; create unique history list and all servers list
    (dolist (servers marketdata-gateway-query-history)
      (pushnew servers query-history :test 'string=)
      (dolist (server (split-string servers))
        (pushnew server all-servers :test 'string=)))
    (setq query-history (nreverse query-history))
    (setq all-servers (sort all-servers 'string-lessp))
    ;; last history
    (setq last-history (car marketdata-gateway-query-history))
    (setq last-history-servers (split-string (car marketdata-gateway-query-history)))
    ;; use local variables to store checkbox server widgets
    (make-local-variable 'widget-checkboxes)
    (setq widget-checkboxes nil)
    (dolist (server all-servers)
      (let ((name (intern (concat "widget-checkbox-" server))))
        (make-local-variable name)
        (push name widget-checkboxes)))
    ;; use local variables to store batch count
    (make-local-variable 'widget-batch)
    (setq widget-batch 1)
    ;; add header
    (widget-insert "MarketData Gateway Query\n\n")
    ;; add buffer name
    (widget-insert (concat "Post Buffer: " post-buffer-name "\n\n"))
    ;; add recent history options
    (if batch
        (progn
          (widget-insert (format "Recent Queries (batch: %s):\n\n" batch))
          ;; add switch to normal mode button
          (widget-insert "  ")
          (widget-create 'push-button
                         :notify `(lambda (&rest ignore)
                                    (kill-buffer nil)
                                    (marketdata-gateway-menu ,server nil))
                         "Normal Query"))
      (progn
        (widget-insert "Recent Queries (normal):\n\n")
        ;; add switch to batch mode button
        (widget-insert "  ")
        (widget-create 'push-button
                       :notify `(lambda (&rest ignore)
                                  (kill-buffer nil)
                                  (marketdata-gateway-menu ,server marketdata-gateway-batch-size))
                       "Batch Query")))
    (widget-insert "\n\n")
    ;; add recent history list
    (do ((servers query-history (cdr servers))
         (cnt 1 (1+ cnt)))
        ((or (not servers) (> cnt 10)))
      (widget-insert "  ")
      (widget-create 'push-button
                     :value (car servers)
                     :notify `(lambda (widget &rest ignore)
                                (let ((value (widget-value widget)))
                                  (kill-buffer nil)
                                  (marketdata-gateway-query value ,batch))))
      (widget-insert "\n"))
    (widget-insert "\n")
    ;; server selection
    (widget-insert "Select Servers:\n\n")
    ;; add all/none buttons
    (widget-insert "  ")
    (widget-create 'push-button
                   :notify `(lambda (&rest ignore)
                              ;; set current servers to all
                              (widget-value-set widget-servers ,(list-to-string all-servers))
                              ;; check all server checkboxes
                              (dolist (widget (list ,@widget-checkboxes))
                                (widget-value-set widget t))
                              (widget-setup))
                   "All")
    (widget-insert "  ")
    (widget-create 'push-button
                   :notify `(lambda (&rest ignore)
                              ;; set current servers to none
                              (widget-value-set widget-servers "")
                              ;; uncheck all server checkboxes
                              (dolist (widget (list ,@widget-checkboxes))
                                (widget-value-set widget nil))
                              (widget-setup))
                   "None")
    (widget-insert "\n\n")
    ;; add checkbox list
    (dolist (server all-servers)
      (widget-insert "  ")
      (let ((widget
             (widget-create
              'checkbox
              :notify (lambda (widget &rest ignore)
                        (let ((value (widget-value widget))
                              (server (widget-get widget :server))
                              (servers (split-string (widget-value widget-servers))))
                          (if value
                              (unless (member server servers)
                                (widget-value-set widget-servers (list-to-string (sort (cons server servers) 'string-lessp))))
                            (widget-value-set widget-servers (list-to-string (remove server servers))))
                          (widget-setup)))
              (member server last-history-servers))))
        (widget-put widget :server server)
        (widget-insert (concat " " server "\n"))
        (funcall `(lambda () (setq ,(intern (concat "widget-checkbox-" server)) widget)))))
    (widget-insert "\n")
    ;; free form entry
    (widget-insert "Servers: ")
    (setq widget-servers
          (widget-create 'editable-field
                         last-history))
    (widget-insert "\n")
    ;; add timeout
    (widget-insert "Timeout: ")
    (widget-create 'editable-field
                   :size 10
                   :notify (lambda (widget &rest ignore)
                             (let ((timeout (widget-value widget)))
                               (setq marketdata-gateway-request-timeout (string-to-number timeout))))
                   (number-to-string marketdata-gateway-request-timeout))
    (widget-insert "  ")
    ;; add batch size
    (widget-insert "Batch Size: ")
    (widget-create 'editable-field
                   :size 10
                   :notify (lambda (widget &rest ignore)
                             (let ((batch (widget-value widget)))
                               (setq marketdata-gateway-batch-size (string-to-number batch))))
                   (number-to-string marketdata-gateway-batch-size))
    (widget-insert "\n\n")
    ;; add submit button
    (widget-create 'push-button
                   :notify (lambda (&rest ignore)
                             (let ((servers (widget-value widget-servers)))
                               (kill-buffer nil)
                               (marketdata-gateway-query servers)))
                   "Submit Query")
    (widget-insert "  ")
    ;; add submit batch button
    (widget-create 'push-button
                   :notify (lambda (&rest ignore)
                             (let ((servers (widget-value widget-servers)))
                               (kill-buffer nil)
                               (marketdata-gateway-query servers marketdata-gateway-batch-size)))
                   "Submit Batch")
    (widget-insert "  ")
    ;; add configure button
    (widget-create 'push-button
                   :notify (lambda (&rest ignore)
                             (kill-buffer nil)
                             (customize-group "marketdata-gateway"))
                   "Configure")
    (widget-insert "  ")
    ;; add history reset button
    (widget-create 'push-button
                   :notify `(lambda (&rest ignore)
                              (kill-buffer nil)
                              (marketdata-gateway-query-history-reset)
                              (marketdata-gateway-menu ,server ,batch))
                   "Reset History")
    (widget-insert "\n")
    ;; final setup
    (use-local-map widget-keymap)
    (widget-setup)
    (switch-to-buffer buffer)
    (goto-char (point-min))
    (widget-forward 2)
    ))

;; delete temp directory
(defun marketdata-gateway-delete-directory (dir)
  "Delete all files in DIR and DIR itself."
  (when (file-exists-p dir)
    (dolist (file (directory-files dir t))
      (unless (string-match "^\\." (file-name-nondirectory file))
        (delete-file file)))))

;; create and clean temp directory
(defun marketdata-gateway-create-directory (dir)
  "Delete the files in DIR, if it exists, else create it."
  ;; delete dir
  (when (file-exists-p dir)
    (marketdata-gateway-delete-directory dir))
  ;; create dir
  (unless (file-exists-p dir)
    (make-directory dir)))

;; test and rename gzip file
(defun marketdata-gateway-rename-gzip-file (file)
  "Test if FILE is a GZIP file, and if it is, rename it with a
`.gz' extension.
\nReturn the new file name."
  (let ((new-file file))
    (when (string-match
           "gzip compressed data"
           (shell-command-to-string (concat "file \"" file "\"")))
      (setq new-file (concat file ".gz"))
      (rename-file file new-file))
    new-file))

;; join strings delimiter
(defun join-strings-delimiter (lst &optional delim)
  "Convert LST of strings into a single string.
\nUse optional DELIM as a delimiter."
  (unless delim
    (setq delim ""))
  (reduce #'(lambda (x y) (concat x delim y)) lst))

;; format time
(defun marketdata-gateway-format-time (time)
  "Return TIME formatted as a string.

TIME is either a list in the format returned by `current-time' or
a floating point number as follows:

  (SECONDS-HI16BITS SECONDS-LOW16BITS MICROSECONDS)
  SECONDS.MICROSECONDS"
  (let (total-seconds
        microsecs)
    (if (listp time)
        (setq total-seconds (+ (* (first time) 65536) (second time))
              microsecs (third time))
      (setq total-seconds (truncate time)
            microsecs (* (- time total-seconds) 1000000)))
    (let* ((hours (floor (/ total-seconds 3600)))
           (mins (floor (/ (- total-seconds (* hours 3600)) 60)))
           (secs (- total-seconds (* hours 3600) (* mins 60)))
           ;; convert microsecs into a string with max length of 3
           (microsecs-string (substring (concat (number-to-string microsecs) "000") 0 3))
           ;; format time
           (time-string (concat
                         (if (> hours 0)
                             (format "%2d:%02d:%02d.%s" hours mins secs microsecs-string)
                           (if (> mins 0)
                               (format "   %2d:%02d.%s" mins secs microsecs-string)
                             (format "      %2d.%s" secs microsecs-string))))))
      time-string)))

;; gateway query
(defun marketdata-gateway-query (&optional server batch)
  "Submit current buffer XML as a SOAP request to a MarketData
Gateway SERVER on the appropriate port.

Requests are created in the `marketdata-gateway-tmp-dir-request'
directory.  Responses are written to files in the
`marketdata-gateway-tmp-dir-response' directory.

If SERVER is nil, the user is prompted for one or a list of them,
with history available and seeded with
`marketdata-gateway-servers'.

If BATCH is non-nil, a batch test is performed where each server
is queried BATCH number of times.  If BATCH < 1, then the test is
run until canceled (with \\[keyboard-quit]).

If `marketdata-gateway-show-output' is t, a buffer is opened for
every response file, otherwise, a `dired' buffer opened on the
`marketdata-gateway-tmp-dir-response' directory.

If `marketdata-gateway-format-output' is t, all output xml is
formatted to make it easier for human consumption."
  (interactive
   (let ((hist (copy-list marketdata-gateway-query-history)))
     (list (read-from-minibuffer "Server(s) to query: "
                                 "" nil nil
                                 'hist))))
  ;; check for required emacs version
  (if (>= emacs-major-version 22)
      ;; check for server parameter
      (if (and server (not (equal server "")) (not (equal server '(""))))
          (let ((servers server))
            ;;    (hist marketdata-gateway-query-history))
            ;; if servers is `all' then substitute full list
            (when (equal servers "all")
              (setq servers (mapcar #'car marketdata-gateway-servers)))
            ;; convert string server to list
            (when (stringp servers)
              (setq servers (split-string servers)))
            ;; update server history
            (marketdata-gateway-query-history-push servers)
            ;; (when (listp servers)
            ;;       (setq server (join-strings-delimiter servers " "))
            ;;       (when (member server hist)
            ;;         (setq hist (remove server hist)))
            ;;       (push server hist)
            ;;       (setq marketdata-gateway-query-history hist))
            ;; main process block
            (let ((xml-buffer (window-buffer)) ; xml buffer
                  status-buffer                ; status buffer
                  processes             ; list of asynchronous processes
                  times                 ; timing information
                  batches               ; batch test batch count
                  recents               ; batch test recent times
                  averages              ; batch test average times
                  dired-buffer          ; response dired buffer
                  desc                  ; gateway description
                  port)                 ; gateway port
              ;; wrap main block in error trap to make sure cleanup occurs
              (condition-case err
                  (progn
                    ;; create request dir
                    (marketdata-gateway-create-directory marketdata-gateway-tmp-dir-request)
                    ;; create response dir
                    (marketdata-gateway-create-directory marketdata-gateway-tmp-dir-response)
                    ;; open dired buffer
                    (unless batch
                      (dired marketdata-gateway-tmp-dir-response)
                      (setq dired-buffer (current-buffer)))
                    ;; loop through servers
                    (dolist (server servers)
                      ;; switch to xml buffer
                      (set-buffer xml-buffer)
                      (message "Querying server: %s" server)
                      (let ((data (buffer-substring-no-properties (point-min) (point-max)))
                            (name (concat "marketdata-gateway-process-" server))
                            (buffer (marketdata-gateway-output-buffer-name server))
                            (file (concat marketdata-gateway-tmp-dir-request "/emacs-marketdata-gateway-buffer-" server))
                            cmd
                            args)
                        ;; if server is in marketdata-gateway-servers resolve to full server name
                        (when (assoc server marketdata-gateway-servers)
                          (setq server (cdr (assoc server marketdata-gateway-servers))))
                        ;; create temporary buffer to modifiy data in
                        (with-temp-buffer
                          (insert data)
                          ;; get gateway description and port, if not already set
                          (unless (and desc port)
                            (let ((info (marketdata-gateway-info)))
                              (setq desc (cdr (assq :name info)))
                              (setq port (number-to-string (cdr (assq :port info))))))
                          ;; prepare data
                          (marketdata-gateway-prepare-data server port)
                          ;; save buffer to temporary file
                          (write-region nil nil file))
                        ;; build request command (output to files)
                        ;; note: wget -nv parameter must be given to prevent status characters
                        ;; from being inserted into the output
                        (setq args (concat
                                    "\"-nv\""
                                    " \"--tries=1\""
                                    " \"--timeout=" (shell-quote-argument (number-to-string marketdata-gateway-request-timeout)) "\""
                                    " \"--post-file=" (shell-quote-argument file) "\""
                                    " \"-O\" \"" (marketdata-gateway-output-file-name buffer) "\""
                                    " \"http://" (shell-quote-argument server)
                                    ":" (shell-quote-argument port) "\""))
                        ;;(message "wget %s" args)
                        ;; create request call
                        ;;(setq cmd (concat "(call-process \"wget\" nil buffer nil " args ")"))
                        (setq cmd (concat "(start-process \"" name "\" nil \"wget\" " args ")"))
                        ;; submit request and track process/buffer/file
                        ;;(message cmd)
                        ;;(push (list (eval (read cmd)) server buffer file (current-time) cmd) processes)
                        (push (cons name (list (eval (read cmd)) server buffer file (current-time) cmd)) processes)
                        ))
                    ;; reverse processes order
                    (setq processes (nreverse processes))
                    ;;(message "%S" processes)
                    ;; setup buffer
                    (setq status-buffer (generate-new-buffer marketdata-gateway-process-status-buffer-name))
                    (switch-to-buffer status-buffer)
                    ;; turn off undo
                    (buffer-disable-undo status-buffer)
                    ;; initialize times
                    (dolist (name (mapcar 'car processes))
                      (push (cons name (list 0 0 0)) times))
                    ;; batch test setup
                    (when batch
                      (dolist (name (mapcar 'car processes))
                        ;; initialize batches
                        (push (cons name 0) batches)
                        ;; initialize recent times
                        (push (cons name 0) recents)
                        ;; initialize average times
                        (push (cons name 0) averages)))
                    ;; loop until all processes have finished
                    (let ((running t)   ; still running
                          (batch-count (when batch
                                        (if (< batch 1)
                                            "infinite"
                                          (number-to-string batch))))) ; pretty version of batch count
                      (while running
                        (setq running nil)
                        (setq buffer-read-only nil)
                        (erase-buffer)
                        ;; output status
                        (insert (concat "MarketData Gateway Query Process Status\n\n"))
                        (if batch
                            (progn
                              (insert (concat "Gateway: " desc "  Port: " port "  Batch Count: " batch-count
                                              "  Timeout: " (number-to-string marketdata-gateway-request-timeout) "\n\n"))
                              (insert "Server                                  Status  Request Time  Recent Time   Average Time  Batch\n")
                              (insert "--------------------------------------  ------  ------------  ------------  ------------  -----\n"))
                          (progn
                            (insert (concat "Gateway: " desc "  Port: " port
                                            "  Timeout: " (number-to-string marketdata-gateway-request-timeout) "\n\n"))
                            (insert "Server                                  Status  Request Time\n")
                            (insert "--------------------------------------  ------  ------------\n")))
                        ;; loop through processes
                        (dolist (alst processes)
                          (let* ((lst (cdr alst))
                                 (name (car alst))
                                 (process (first lst))
                                 (server (second lst))
                                 (status (process-status process))
                                 (time (fifth lst))
                                 (time-diff (cdr (assoc name times)))
                                 (run (when batch (cdr (assoc name batches))))
                                 (avg (when batch (cdr (assoc name averages))))
                                 (microsecs (third time-diff))
                                 (total-seconds (+ (* (first time-diff) 65536) (second time-diff)))
                                 (hours (floor (/ total-seconds 3600)))
                                 (mins (floor (/ (- total-seconds (* hours 3600)) 60)))
                                 (secs (- total-seconds (* hours 3600) (* mins 60))))
                            ;; if doing a batch test, restart finished processes
                            (when (and batch (< run batch) (not (eq status 'run)))
                              ;; update run count
                              (setcdr (assoc name batches) (1+ run))
                              ;; get real time
                              (let ((mic microsecs)
                                    (sec total-seconds))
                                (while (>= mic 1)
                                  (setq mic (/ mic 10.0)))
                                (setq sec (+ sec mic))
                                ;; update recent time
                                (setcdr (assoc name recents) sec)
                                ;; update average time
                                (if (= run 0)
                                    (setq avg sec)
                                  (setq avg (/ (+ (* avg run) sec) (1+ run))))
                                (setcdr (assoc name averages) avg))
                              ;; restart process
                              (let ((buffer (third lst))
                                    (file (fourth lst))
                                    (cmd (sixth lst)))
                                ;; update time
                                (setq time (current-time))
                                ;; replace process
                                (setcdr alst (list (eval (read cmd)) server buffer file time cmd))
                                ;; set status to run
                                (setq status 'run)))
                            ;; if processes is running
                            (when (eq status 'run)
                              ;; calculate and set time difference
                              (setcdr (assoc name times) (time-subtract (current-time) time))
                              ;; set to not done
                              (setq running t))
                            ;; output process status
                            (let ((server (if (> (length server) 38)
                                              (setq server (substring server 0 38))
                                            server))
                                  (time-disp (marketdata-gateway-format-time time-diff))
                                  (rec-disp (when batch (marketdata-gateway-format-time (cdr (assoc name recents)))))
                                  (avg-disp (when batch (marketdata-gateway-format-time avg))))
                              (if batch
                                  (insert (format "%-39s %-7S %s  %s  %s  %5d\n" server status time-disp rec-disp avg-disp run))
                                (insert (format "%-39s %-7S %s\n" server status time-disp))))))
                        (setq buffer-read-only t)
                        ;; draw screen and wait 0.1 second
                        (sit-for 0.1)
                        ))
                    ;; format responses
                    (setq buffer-read-only nil)
                    (if batch
                        (insert "\n")
                      (insert "\nFormatting responses..."))
                    (setq buffer-read-only t)
                    ;; draw screen
                    (sit-for 0)
                    ;; delete temp request files and directory
                    (marketdata-gateway-delete-directory marketdata-gateway-tmp-dir-request)
                    ;; process result files
                    (unless batch
                      ;; loop through all processes (in reverse order)
                      (dolist (alst (reverse processes))
                        (let* ((lst (cdr alst))
                               (process (first lst))
                               (buffer (third lst))
                               file)
                          ;; cleanup results
                          ;; get response file name and add .gz to it if it is a gzip response
                          (setq file (marketdata-gateway-rename-gzip-file (marketdata-gateway-output-file-name buffer)))
                          ;; load response file
                          (find-file file)
                          ;; reformat if buffer is not too large and `marketdata-gateway-format-output' is t
                          (when marketdata-gateway-format-output
                            ;; use `font-lock-maximum-size' to determine what "too large" is
                            (when (<= (buffer-size) font-lock-maximum-size)
                              (marketdata-gateway-xml-reformat)))
                          ;; save buffer
                          (save-buffer)
                          ;; close buffer unless `marketdata-gateway-format-output' is t
                          (unless marketdata-gateway-show-output
                            (kill-this-buffer))
                          ;; update status
                          (switch-to-buffer status-buffer)
                          (setq buffer-read-only nil)
                          (insert ".")
                          (setq buffer-read-only t)
                          ;; draw screen
                          (sit-for 0)
                          )))
                    ;; kill dired buffer
                    (unless batch
                      (kill-buffer dired-buffer))
                    ;; switch to status buffer
                    (switch-to-buffer status-buffer)
                    (setq buffer-read-only nil)
                    (insert "done\n")
                    (setq buffer-read-only t)
                    (delete-other-windows)
                    ;; open dired buffer
                    (unless (or marketdata-gateway-show-output batch)
                      (dired marketdata-gateway-tmp-dir-response)
                      (switch-to-buffer status-buffer))
                    ;; kill status buffer
                    ;;(kill-buffer status-buffer)
                    )
                ;; cleanup on trapped error
                (error
                 ;; delete temp request files and dir
                 (marketdata-gateway-delete-directory marketdata-gateway-tmp-dir-request)
                 ;; open dired buffer
                 (unless batch
                   (dired marketdata-gateway-tmp-dir-response))
                 (switch-to-buffer status-buffer)
                 ;; pass thru normal error message
                 (message "%s" (error-message-string err))))))
        ;; query user for server
        (marketdata-gateway-menu server batch))
    ;; inform user of emacs version limitation
    (error "This function requires Emacs version 22 or higher")))

;; gateway query batch
(defun marketdata-gateway-query-batch (&optional server batch)
  "Run `marketdata-gateway-query' in batch mode.
\nIf BATCH is nil, `marketdata-gateway-batch-size' is used."
  (interactive
   (let ((hist (copy-list marketdata-gateway-query-history)))
     (list (read-from-minibuffer "Server(s) to query: "
                                 "" nil nil
                                 'hist))))
  (if batch
      (marketdata-gateway-query server batch)
    (marketdata-gateway-query server marketdata-gateway-batch-size)))

;; xml reformat
(defun marketdata-gateway-xml-reformat ()
  "Reformat XML buffer.
\nConvert poorly formatted XML into something better."
  (xml-mode)
  (let ((xml-eol "[ \t]*\n[ \t]*")
        (xml-tag-end-regexp ">")
        (xml-close-tag-regexp "</")
        (xml-block-regexp "<[^>]*>[^<]*</[^>]*>"))
    (save-excursion
      ;; preserve search data
      (save-match-data
        ;; remove all existing EOL characters
        (goto-char (point-min))
        (while (re-search-forward xml-eol nil t)
          (replace-match "" nil nil))
        ;; move down code adding newlines were needed
        (goto-char (point-min))
        (while (re-search-forward xml-tag-end-regexp nil t)
          (insert "\n")
          (when (and
                 (looking-at xml-block-regexp)
                 (not (looking-at xml-close-tag-regexp)))
            (re-search-forward xml-block-regexp nil t)
            (forward-char -1)))
        ;; indent buffer
        (indent-region (point-min) (point-max))
        ))))

(provide 'marketdata-gateway)

;;; marketdata-gateway.el ends here