Update to sorcery devel version

21 files changed, 1068 insertions, 343 deletions

diff --git a/var/lib/sorcery/modules/build_api/api1 b/var/lib/sorcery/modules/build_api/api1
index 0605919..63146c5 100644
--- a/var/lib/sorcery/modules/build_api/api1
+++ b/var/lib/sorcery/modules/build_api/api1
@@ -95,6 +95,8 @@ function run_pre_build() {
   if  [  -x  $SCRIPT_DIRECTORY/PRE_BUILD  ];  then
     debug "build_api/api1" "run_pre_build() - Prebuild script exists, now sourcing "
     . $SCRIPT_DIRECTORY/PRE_BUILD
+  elif [[ "$PRE_BUILD_CMD" ]] ; then
+    $PRE_BUILD_CMD
   else
     debug "build_api/api1" "run_pre_build() - Prebuild script not found, using default"
     default_pre_build
@@ -133,6 +135,8 @@ function run_build()  {
     persistent_load &&
     if  [  -x  $SCRIPT_DIRECTORY/BUILD  ];  then
       .  $SCRIPT_DIRECTORY/BUILD
+    elif [[ "$BUILD_CMD" ]] ; then
+      $BUILD_CMD
     else
       default_build
     fi &&
@@ -162,6 +166,8 @@ function run_post_build() {
   persistent_load &&
   if  [  -x  $SCRIPT_DIRECTORY/POST_BUILD  ];  then
            . $SCRIPT_DIRECTORY/POST_BUILD
+  elif [[ "$POST_BUILD_CMD" ]] ; then
+    $POST_BUILD_CMD
   else
     default_post_build
   fi
@@ -185,6 +191,8 @@ function run_post_install() {
   if    [  -x  $SCRIPT_DIRECTORY/POST_INSTALL  ]
   then
     . $SCRIPT_DIRECTORY/POST_INSTALL
+  elif [[ "$POST_INSTALL_CMD" ]] ; then
+    $POST_INSTALL_CMD
   fi &&
   persistent_save
 }
diff --git a/var/lib/sorcery/modules/build_api/api2 b/var/lib/sorcery/modules/build_api/api2
index e533f8b..3fe61d5 100644
--- a/var/lib/sorcery/modules/build_api/api2
+++ b/var/lib/sorcery/modules/build_api/api2
@@ -97,10 +97,10 @@ function run_pre_build() {
   verify_sources &&
   persistent_load &&
   if  [  -x  $SCRIPT_DIRECTORY/PRE_BUILD  ];  then
-    debug "build_api/api2" "run_pre_build() - Prebuild script exists, now sourcing "
     . $SCRIPT_DIRECTORY/PRE_BUILD
+  elif [[ "$PRE_BUILD_CMD" ]] ; then
+    $PRE_BUILD_CMD
   else
-    debug "build_api/api2" "run_pre_build() - Prebuild script not found, using default"
     default_pre_build
   fi &&
   persistent_save
@@ -133,6 +133,8 @@ function run_build()  {
   persistent_load &&
   if  [  -x  $SCRIPT_DIRECTORY/BUILD  ];  then
     .  $SCRIPT_DIRECTORY/BUILD
+  elif [[ "$BUILD_CMD" ]] ; then
+    $BUILD_CMD
   else
     default_build
   fi &&
@@ -157,6 +159,8 @@ function run_pre_install() {
   persistent_load &&
   if  [  -x  $SCRIPT_DIRECTORY/PRE_INSTALL  ];  then
     .  $SCRIPT_DIRECTORY/PRE_INSTALL 
+  elif [[ "$PRE_INSTALL_CMD" ]] ; then
+    $PRE_INSTALL_CMD
   else
     default_pre_install
   fi &&
@@ -178,6 +182,8 @@ function run_install() {
   persistent_load &&
   if  [  -x  $SCRIPT_DIRECTORY/INSTALL  ];  then
     .  $SCRIPT_DIRECTORY/INSTALL 
+  elif [[ "$INSTALL_CMD" ]] ; then
+    $INSTALL_CMD
   else
     default_install
   fi &&
@@ -201,6 +207,8 @@ function run_post_install() {
   persistent_load &&
   if    [  -x  $SCRIPT_DIRECTORY/POST_INSTALL  ]; then
     . $SCRIPT_DIRECTORY/POST_INSTALL
+  elif [[ "$POST_INSTALL_CMD" ]] ; then
+    $POST_INSTALL_CMD
   else
     default_post_install
   fi &&
@@ -220,6 +228,8 @@ function run_final() {
   if    [  -x  $SCRIPT_DIRECTORY/FINAL  ]
   then
     . $SCRIPT_DIRECTORY/FINAL
+  elif [[ "$FINAL_CMD" ]] ; then
+    $FINAL_CMD
   fi &&
   persistent_save
 }
diff --git a/var/lib/sorcery/modules/build_api/common b/var/lib/sorcery/modules/build_api/common
index 338e58d..db84876 100644
--- a/var/lib/sorcery/modules/build_api/common
+++ b/var/lib/sorcery/modules/build_api/common
@@ -212,10 +212,12 @@ function run_spell_success() {
 
     # update depends info
     debug "build_api/common" "Merging depends info"
+
     local t_DEPENDS_STATUS=$(lock_start_transaction $DEPENDS_STATUS)
     # none of the old values are valid, only the new, uncommitted values are
     remove_depends_status $t_DEPENDS_STATUS $SPELL
-    local spell_depends=$(hash_get uncommitted_hash $SPELL)
+    local spell_depends spell_sub_depends spell_rsub_depends
+    get_uncommitted_depends_file $SPELL spell_depends
     if [ -e $spell_depends ] ; then
       cat  $spell_depends >> $t_DEPENDS_STATUS
     fi
@@ -226,6 +228,17 @@ function run_spell_success() {
     test -e "$ABANDONED_DEPENDS/$SPELL" &&
     rm -f $ABANDONED_DEPENDS/$SPELL &>/dev/null
 
+    local t_SUB_DEPENDS_STATUS=$(lock_start_transaction $SUB_DEPENDS_STATUS)
+    # none of the old values are valid, only the new, uncommitted values are
+    remove_sub_depends $t_SUB_DEPENDS_STATUS "" "$SPELL"
+    local spell_sub_depends
+    get_uncommitted_sub_depends_file $SPELL spell_sub_depends
+    if [ -e $spell_sub_depends ] ; then
+      cat  $spell_sub_depends >> $t_SUB_DEPENDS_STATUS
+    fi
+    lock_commit_transaction $SUB_DEPENDS_STATUS
+    get_uncommitted_rsub_depends_file $SPELL spell_rsub_depends
+
     # compile log
     create_compile_log
     view_compile_log
@@ -235,10 +248,12 @@ function run_spell_success() {
       message "failed to make a tablet directory: $tablet_dir" \
               "this may be a bug"
       unset tablet_dir
+      local TABLET_IGNORE=yes
     else
       message  "${MESSAGE_COLOR}Creating tablet in directory"        \
                "${FILE_COLOR}${tablet_dir}${DEFAULT_COLOR}"
       tablet_install_spell_files $tablet_dir
+      local TABLET_SPELL_DIR=$tablet_dir
 
       # this data is now doubly stale, remove it
       rm  -f $ABANDONED_PERSIST/$SPELL.p
@@ -273,7 +288,8 @@ function run_spell_success() {
     create_cache_archive $TMP_INST_LOG $CACHE $CACHE_COMP
 
     report_install
-    rm  -f $IW_LOG $C_LOG $TMP_INST_LOG $TMP_MD5_LOG $spell_depends
+    rm  -f $IW_LOG $C_LOG $TMP_INST_LOG $TMP_MD5_LOG $spell_depends \
+           "$spell_sub_depends" "$spell_rsub_depends"
 }
 
 #---------------------------------------------------------------------
@@ -297,7 +313,8 @@ function run_spell_failure() {
 
     # put the answers somewhere where they can still be used later
     mkdir -p $ABANDONED_DEPENDS
-    local spell_depends=$(hash_get uncommitted_hash $SPELL)
+    local spell_depends
+    get_uncommitted_depends_file $SPELL spell_depends
     [ -e $spell_depends ] && mv $spell_depends $ABANDONED_DEPENDS/$SPELL
 
     # This may have been locked if there was a failure during the install
@@ -308,6 +325,11 @@ function run_spell_failure() {
 
     rm -f $IW_LOG $C_LOG
 
+    local spell_sub_depends spell_rsub_depends
+    get_uncommitted_sub_depends_file $SPELL spell_sub_depends
+    get_uncommitted_rsub_depends_file $SPELL spell_rsub_depends
+    rm -f "$spell_sub_depends" "$spell_rsub_depends"
+
     [[  $CLEAN_SOURCE == on ]] && rm_source_dir
     CAST_EXIT_STATUS=1
 }
diff --git a/var/lib/sorcery/modules/dl_handlers/dl_file b/var/lib/sorcery/modules/dl_handlers/dl_file
index 28e0a1d..aa36001 100644
--- a/var/lib/sorcery/modules/dl_handlers/dl_file
+++ b/var/lib/sorcery/modules/dl_handlers/dl_file
@@ -33,7 +33,7 @@ function dl_file_get() {
   [[ $target ]] || return 255
 
   for url in $url_list; do
-    FILENAME=`url_file_crack $url`  &&
+    FILENAME=`url_crack $url`  &&
     test -f  "$FILENAME" &&
     test -r  "$FILENAME" &&
     cp -a $FILENAME $target && break
diff --git a/var/lib/sorcery/modules/dl_handlers/dl_svn b/var/lib/sorcery/modules/dl_handlers/dl_svn
index 6d50484..c0a4cef 100644
--- a/var/lib/sorcery/modules/dl_handlers/dl_svn
+++ b/var/lib/sorcery/modules/dl_handlers/dl_svn
@@ -35,32 +35,19 @@ function dl_svn_get () {
 
   for url in $url_list; do
     local URL SVN_ROOT SVN_MODULE SVN_TAG
-    url_svn_crack $url
+    url_crack "$url" "$hints"
 
     if test -d $target; then
       message "${MESSAGE_COLOR}Running svn update...${DEFAULT_COLOR}"
+      echo svn update -q --non-interactive -r $SVN_TAG $target
       svn update -q --non-interactive -r $SVN_TAG $target
       rc=$?
       eval "$dl_target=\"$target\""
     else
       message "${MESSAGE_COLOR}Running svn checkout...${DEFAULT_COLOR}"
-      # hard-coded http :-( ....
-      case $SVN_TYPE in
-      	 svn) 
-		message "svn checkout -r $SVN_TAG http://$SVN_ROOT $SVN_MODULE"
-		svn checkout -r $SVN_TAG http://$SVN_ROOT $SVN_MODULE 
-      		rc=$?
-	   ;;
-	svns)
-		message "svn checkout -r $SVN_TAG https://$SVN_ROOT $SVN_MODULE"
-		svn checkout -r $SVN_TAG https://$SVN_ROOT $SVN_MODULE 
-      		rc=$?
-	   ;;
-	   *) 
-	   	message "unknow url type"
-		rc=2
-	   ;;
-      esac
+      echo svn checkout -r $SVN_TAG $SVN_ROOT $SVN_MODULE
+      svn checkout -r $SVN_TAG $SVN_ROOT $SVN_MODULE
+      rc=$?
       eval "$dl_target=\"$SVN_MODULE\""
     fi
     [[ $rc == 0 ]] && break
diff --git a/var/lib/sorcery/modules/libapi b/var/lib/sorcery/modules/libapi
index dbb65b1..95ece02 100644
--- a/var/lib/sorcery/modules/libapi
+++ b/var/lib/sorcery/modules/libapi
@@ -19,6 +19,7 @@
 #---------------------------------------------------------------------
 # functions defined in here: (in alphabetical order)
 #    acquire_src            (libsummon)
+#    altruistic_sub_depends (libdepends)
 #    config_query           (libmisc)
 #    config_query_list      (libmisc)
 #    config_query_option    (libmisc)
@@ -39,6 +40,7 @@
 #    get_source_nums        (libmisc)
 #    guess_compressor       (libunpack)
 #    handle_changed_config  (libresurrect)
+#    is_depends_enabled     (libstate)
 #    installed_version      (libgrimoire)   
 #    install_config_file    (libresurrect)
 #    invoke_installwatch    (libtrack)
@@ -64,6 +66,7 @@
 #    requires               (libdepends)
 #    rm_source_dir          (libgrimoire)
 #    sedit                  (libmisc)       
+#    sub_depends            (libdepends)
 #    spell_exiled           (libstate)
 #    spell_held             (libstate)
 #    spell_installed        (libstate)
@@ -119,6 +122,19 @@ function acquire_src() {
 }
 
 #---------------------------------------------------------------------
+## @Type API
+## @params Name of a sub-depends that the current spell is providing.
+## @See <@function var.lib.sorcery.modules.libdepends.html,real_altruistic_sub_depends> for more details.
+##
+## Altruistically provide a sub-depends.
+##
+#---------------------------------------------------------------------
+function altruistic_sub_depends() {
+    debug "libapi" "$FUNCNAME - $*"
+    real_altruistic_sub_depends "$@"
+}
+
+#---------------------------------------------------------------------
 ## @See <@function var.lib.sorcery.modules.libmisc.html,real_config_query> for more details.
 ## @param config file variable
 ## @param question
@@ -481,6 +497,19 @@ function handle_changed_config() {
 #---------------------------------------------------------------------
 ## @Type API
 ## @param spell
+## @param dependency
+## @See <@function var.lib.sorcery.modules.libstate.html,is_depends_enabled> for more details.
+## Returns whether or not a given dependency is enabled
+##
+#---------------------------------------------------------------------
+function is_depends_enabled() {
+    debug "libapi" "is_depends_enabled - $*"
+    real_is_depends_enabled "$@"
+}
+
+#---------------------------------------------------------------------
+## @Type API
+## @param spell
 ## @See <@function var.lib.sorcery.modules.libgrimoire.html,real_install_version> for more details.
 ## Returns the current version of the given spell
 ##
@@ -934,6 +963,17 @@ function spell_exiled() {
 
 #---------------------------------------------------------------------
 ## @Type API
+## @param spell providing the sub-depends
+## @param name of sub-depends
+## @See <@function var.lib.sorcery.modules.libdepends
+#---------------------------------------------------------------------
+function sub_depends() {
+    debug "libapi" "sub_depends - $*"
+    real_sub_depends "$@"
+}
+
+#---------------------------------------------------------------------
+## @Type API
 ## @param file to uncompress
 ## @See <@function var.lib.sorcery.modules.libunpack.html,real_uncompress> for more details.
 ##
diff --git a/var/lib/sorcery/modules/libcast b/var/lib/sorcery/modules/libcast
index b5a61b1..a7f7c41 100644
--- a/var/lib/sorcery/modules/libcast
+++ b/var/lib/sorcery/modules/libcast
@@ -123,6 +123,38 @@ function run_conflicts() {
 
 }
 
+function query_custom_cflags() {
+  if [[ $PER_SPELL_CFLAGS == on ]]; then
+    query "Would you like to set custom optimizations for this spell?" n ||
+      return 0
+
+    persistent_add USER_SPELL_CFLAGS
+    persistent_add USER_SPELL_CXXFLAGS
+    persistent_add USER_SPELL_LDFLAGS
+    local RESULTS KEY rc
+    while true ; do
+      # input box returns things of the form "RENAMED CFLAGS -O3 -march=foo ..."
+      # the ( ) interprits as an array, then using the magic power of set we
+      # get to use shift and $@ to get things done
+      RESULTS=(`eval dialog --stdout ' --extra-label Adjust --inputmenu \
+                              "Custom Optimizations Menu" \
+                               17 50 9 \
+                               CFLAGS "$USER_SPELL_CFLAGS" \
+                               CXXFLAGS "$USER_SPELL_CXXFLAGS" \
+                               LDFLAGS "$USER_CUSTOM_LDFLAGS"'`)
+      rc=$?
+      [[ $rc == 0 ]] || [[ $rc == 1 ]] && break
+      set ${RESULTS[*]}
+      KEY=$2
+      shift 2
+      case "$KEY" in
+        CFLAGS) USER_SPELL_CFLAGS="$@"  ;;
+        CXXFLAGS) USER_SPELL_CXXFLAGS="$@" ;;
+        LDFLAGS) USER_SPELL_LDFLAGS="$@" ;;
+      esac
+    done
+  fi
+}
 
 #---------------------------------------------------------------------
 ## Asks the user about init and xinetd services
diff --git a/var/lib/sorcery/modules/libdepends b/var/lib/sorcery/modules/libdepends
index 5ac59d3..0d499e9 100644
--- a/var/lib/sorcery/modules/libdepends
+++ b/var/lib/sorcery/modules/libdepends
@@ -112,6 +112,7 @@ function run_other() {
   # ask the questions about xinetd/initd script installation
   persistent_load
   query_services
+  query_custom_cflags
   persistent_save
 # todo:
 #ask about conflicts
@@ -184,9 +185,106 @@ function run_up_triggers() {
   fi
   
 }
-######################END CALLS TO OUTSIDE WORLD########################
 
+#---------------------------------------------------------------------
+## process any sub-depends the current spell is going to provide
+## These come from the live sub-depends table and from spells that
+## have already been processed
+#---------------------------------------------------------------------
+function run_our_sub_depends() {
+  $STD_DEBUG
+  local sub_depends pair
+  # do all sub-depends on us from live-table first, they get priority
+  # in case there is a conflict
+  for triple in $(search_sub_depends "$SUB_DEPENDS_STATUS" '.*' "$SPELL") ; do
+    # parse the line out, eventually these silly variable expansions should
+    # be in functions or benchmarked against explode
+    tmp=${triple%:*};
+    requester=${tmp%:*};
+    sub_dependee=${tmp#*:};
+    sub_depends=${triple##*:}
+
+    process_sub_depends "$requester" "$sub_dependee" "$sub_depends" || return 1
+  done
+
+  # do all sub-depends from sub_dep_r_hash (things that we provide)
+  for pair in $(hash_get sub_dep_r_hash $SPELL) ; do
+    requester=${pair%:*}
+    sub_depends=${pair#*:}
+
+    process_sub_depends "$requester" "$SPELL" "$sub_depends" || return 1
+  done
+}
+
+#---------------------------------------------------------------------
+## run any sub-depends that we requested on behalf of the sub-dependee
+#---------------------------------------------------------------------
+function run_other_sub_depends() {
+  $STD_DEBUG
+  local this_spell=$1
+  local SPELL sub_dependee sub_depends pair
+  # do all sub-depends from sub_dep_f_hash (things we request)
+  for pair in $(hash_get sub_dep_f_hash $this_spell) ; do
+    # FIXME, do this one spell at a time, rather than in whatever order we
+    # see them it will reduce the number of run_details
+    sub_dependee=${pair%:*}
+    sub_depends=${pair#*:}
+    if [[ $(hash_get depends_looked_at $sub_dependee) == done ]] ; then
+      # load the spell and clear out any special variables
+      SPELL=$sub_dependee
+      run_details
+      local NEW_SUB_DEPENDEES=""
+      local NEW_DEPENDS=()
+      process_sub_depends "$this_spell" "$sub_dependee" "$sub_depends" ||
+      return 1
+
+      # this is needed in case processing the sub-depends added a depends
+      # isnt this fun?
+      private_add_depends
+    fi
+  done
+}
+
+#---------------------------------------------------------------------
+## Run a spell's SUB_DEPENDS file for a specific sub-depends.
+## This is idempotent, if a sub-depends has already been processed it will
+## not run the SUB_DEPENDS file again.
+##
+## Expects the sub-dependee to be loaded.
+## If the spell does not have a SUB_DEPENDS file, it is a failure.
+#---------------------------------------------------------------------
+function process_sub_depends() {
+  $STD_DEBUG
+  local requester=$1
+  local sub_dependee=$2
+  local sub_depends=$3
+
+  if test -x $SCRIPT_DIRECTORY/SUB_DEPENDS; then
+    if ! list_find "$(hash_get processed_sub_depends $sub_dependee)" "$sub_depends"; then
+      local rc=0
+
+      local THIS_SUB_DEPENDS=$sub_depends
+      local PROCESSED_SUB_DEPENDS=$(hash_get processed_sub_depends $sub_dependee)
+      persistent_load &&
+      source $SCRIPT_DIRECTORY/SUB_DEPENDS &&
+      persistent_save || return 1
+      hash_append processed_sub_depends $sub_dependee $sub_depends
+    else
+      debug libdepends "SUB_DEPENDS: $sub_dependee needs to provide $sub_depends but has already been processed"
+    fi
+  else
+    # sub-depends file doesnt exist, something messed up somewhere
+    return 1
+  fi
+  # add the information to the uncommitted depends file
+  local sub_depends_file rsub_depends_file
+  get_uncommitted_sub_depends_file "$sub_dependee" sub_depends_file || return 1
+  add_sub_depends "$sub_depends_file" "$requester" "$sub_dependee" "$sub_depends"
+  get_uncommitted_rsub_depends_file "$requester" rsub_depends_file || return 1
+  add_sub_depends "$rsub_depends_file" "$requester" "$sub_dependee" "$sub_depends"
+}
 
+######################END CALLS TO OUTSIDE WORLD########################
 
 #---------------------------------------------------------------------
 ## Create a map of spells to their dependent spells.
@@ -300,11 +398,14 @@ function compute_uninstalled_depends()
       debug "libdepends" "already looked at ${spells[$_idx]}, skipping"
     fi
   done
-  # we no longer need this, no sense in keeping it around
+  # we no longer need these, no sense in keeping them around
   hash_unset depends_looked_at
+  hash_unset sub_dep_f_hash
+  hash_unset sub_dep_r_hash
+  hash_unset sub_depends_process
+  hash_unset processed_sub_depends
 
-  # we need this so processes on the other side of make know whats
-  # going on
+  # we need this so processes on the other side of make know whats going on
   hash_export uncommitted_hash
 }
 
@@ -317,9 +418,14 @@ function private_run_depends()
 {
   debug "libdepends" "$FUNCNAME - $*"
   local SPELL=$1
+
+  # special accumulators for the current spell
   local NEW_DEPENDS=""
+  local NEW_SUB_DEPENDEES=""
   local triggerees=""
   local spell_depends
+  local spell_sub_depends
+
   hash_put "depends_looked_at" "$SPELL" "start"
 
   # move this up to compute_uninstalled_depends?
@@ -343,9 +449,12 @@ function private_run_depends()
       return 1
     fi
     get_uncommitted_depends_file $SPELL spell_depends
+    spell_sub_depends=$spell_depends.sub
     if  [  -n  "$RECONFIGURE"  ];  then
       rm  -f  $DEPENDS_CONFIG/$SPELL
-      rm  -f  $DEPENDS_CONFIG/$SPELL.p
+      test -f $DEPENDS_CONFIG/$SPELL.p &&
+      mkdir -p $ABANDONED_PERSIST      &&
+      mv  -f  $DEPENDS_CONFIG/$SPELL.p $ABANDONED_PERSIST/$SPELL.p
     fi
     prepare_spell_config
     SCRIPT_DIRECTORY=`codex_find_spell_by_name $SPELL`
@@ -355,14 +464,20 @@ function private_run_depends()
     run_other $SPELL              &&
     run_depends $SPELL            && 
     run_up_triggers $SPELL        && 
+    run_our_sub_depends $SPELL        &&
 
     # possibly recast things that depend on us if option is set (-B)
     private_upward_depends $SPELL &&
+    private_recast_optionals $SPELL &&
     private_add_triggerees        &&
     private_add_depends           ||
-    { debug "libdepends" "$FUNCNAME: false inside if." ; return 1; }
+    { debug "libdepends" "$FUNCNAME failed to process $SPELL." ; return 1; }
     # no point in keeping the file around if its empty...
-    test -s $spell_depends || rm $spell_depends
+    test -s $spell_depends || rm -f $spell_depends $spell_sub_depends
+
+    # this processes any sub-depends we requested and the sub-dependee
+    # has already been processed, so we must process them on their behalf
+    run_other_sub_depends "$SPELL" || return 1
   else 
     message  "${SPELL_COLOR}${SPELL}${DEFAULT_COLOR}" \
              "${CHECK_COLOR}No work to do.${DEFAULT_COLOR}"
@@ -382,7 +497,7 @@ function private_run_depends()
 #---------------------------------------------------------------------
 function private_should_cast()
 {
-  local decision
+  local each
   # order is important here...
   if ! codex_does_spell_exist $1; then
     return 1
@@ -403,15 +518,16 @@ function private_should_cast()
     # if its being triggered we need to look at it, despite its
     # installed status
     return 0
+  elif [[ "$(hash_get sub_depends_process $1)" != "" ]] ; then
+    return 0
   elif spell_installed $1 ; then
-    # spell is installed and no -R or -B, so dont cast
-    return 1
+    # returns success if want to cast
+    want_lazy_update $1 && return 0 || return 1
   fi
   
   # we must need to install this as we know nothing else about it
   return 0
 }
-
 #---------------------------------------------------------------------
 ## @param Spell name
 ## Find all the spells that depend on the spell given as $1
@@ -433,6 +549,40 @@ function private_upward_depends() {
   fi
 }
 
+#---------------------------------------------------------------------
+## Find spells that optionally depended on the current spell but had the
+## dependency disabled, ask if the user wants to recast the spell.
+#---------------------------------------------------------------------
+function private_recast_optionals() {
+  if [[ $RECAST_OPTIONALS ]] && [[ $RECAST_OPTIONALS != ignore ]] ; then
+    local spell j enabled
+    let j=${#UP_DEPENDS[*]}
+    for spell in $(search_depends_status_exact $DEPENDS_STATUS \
+                 '.*' "$1\(([^:]*)\)\?" off optional '.*' '.*'|cut -f1 -d:); do
+
+      enabled=""
+      message "${SPELL_COLOR}$spell${DEFAULT_COLOR}${CHECK_COLOR}" \
+              "has a disabled optional depends on${DEFAULT_COLOR}" \
+              "${SPELL_COLOR}$1${DEFAULT_COLOR}"
+      if [[ $RECAST_OPTIONALS == always ]] ; then
+        enabled=yes
+      else
+        local default=n
+        [[ $RECAST_OPTIONALS == ask-yes ]] && default=y
+        if query "Recast $spell with dependency enabled?" $default ; then
+          enabled=yes
+        fi
+      fi
+      if [[ $enabled ]] ; then
+        UP_DEPENDS[$j]=$each
+        let j++
+        remove_depends_status $DEPENDS_STATUS $each "${1}\(([^:]*)\)\?"
+      fi
+    done
+  fi
+
+}
+
 ###################BEGIN CALLBACKS FROM OUTSIDE#######################
 
 #---------------------------------------------------------------------
@@ -445,6 +595,11 @@ function private_upward_depends() {
 #---------------------------------------------------------------------
 function real_depends()
 {
+  if [[ "$1" == "-sub" ]] ; then
+    local requested_sub_depends=$2
+    shift 2
+  fi
+
   # see if theres another grimoire
   if [[ "$4" ]] ; then
     local grimoire here nothere current
@@ -514,12 +669,19 @@ function real_depends()
     fi
   fi
 
+  local target_spell
   if ! codex_does_spell_exist $1 &> /dev/null; then
-    work_depends_provider "$@"
+    work_depends_provider "$@" &&
+    if [[ "$requested_sub_depends" ]]; then
+      target_spell=$(get_spell_provider "$SPELL" "$1")
+    fi
   else
-    work_depends_spell "$@"
-  fi
-
+    work_depends_spell "$@" &&
+    target_spell=$1
+  fi &&
+  for sub_depend in $requested_sub_depends; do
+    sub_depends "$target_spell" "$sub_depend" || return 1
+  done
 }
 
 #---------------------------------------------------------------------
@@ -532,6 +694,10 @@ function real_depends()
 #---------------------------------------------------------------------
 function real_optional_depends()
 {
+  if [[ "$1" == "-sub" ]] ; then
+    local requested_sub_depends=$2
+    shift 2
+  fi
 
   # see if theres another grimoire
   if [[ "$5" ]] ; then
@@ -600,10 +766,22 @@ function real_optional_depends()
     fi
   fi
 
+  local target_spell
   if ! codex_does_spell_exist $1 &> /dev/null; then
-    work_optional_depends_provider "$@"
+    work_optional_depends_provider "$@" &&
+    if [[ "$requested_sub_depends" ]]; then
+      target_spell=$(get_spell_provider "$SPELL" "$1")
+    fi
   else
-    work_optional_depends_spell "$@" 
+    work_optional_depends_spell "$@"  &&
+    target_spell=$1
+  fi &&
+  if [[ "$requested_sub_depends" ]] &&
+     [[ "$target_spell" ]] &&
+     is_depends_enabled "$SPELL" "$target_spell"; then
+    for sub_depend in $requested_sub_depends; do
+      sub_depends "$target_spell" "$sub_depend" || return 1
+    done
   fi
 
 }
@@ -905,7 +1083,6 @@ function work_optional_depends_spell()
   private_common_depends "$1" "$install" "optional" "$2" "$3"
 }
 
-
 #---------------------------------------------------------------------
 ## @param  name of return variable
 ## @param  default answer
@@ -923,7 +1100,7 @@ function select_provider()
     shift 3
 
     local each default_char=0 stuff=()
-    local char answer spell
+    local char
 
     # we can only read one character so use every one we can, I dont expect
     # there to be more than 62 providers
@@ -955,22 +1132,9 @@ function select_provider()
       let i++
     done
 
-    message -n "\n${QUERY_COLOR}Which one do you want? " \
-               "[$default_char]$DEFAULT_COLOR "
-    read   -t  $PROMPT_DELAY  -n  1 answer 
-    [[ $answer ]] || answer=$default_char
-    spell="$(hash_get CHAR_TO_SPELL $answer)"
-
-    while [[ ! $spell ]] ; do
-      message -n "\n${QUERY_COLOR}Which one do you want? " \
-                 "[$default_char]$DEFAULT_COLOR "
-      read   -t  $PROMPT_DELAY  -n  1 answer 
-      [[ $answer ]] || answer=$default_char
-      spell=$(hash_get CHAR_TO_SPELL $answer)
-    done
-    echo
+    local msg="\n${QUERY_COLOR}Which one do you want? [$default_char]$DEFAULT_COLOR "
+    select_list_sub "$returnvar" CHAR_TO_SPELL "$msg" "$default_char"
     hash_unset CHAR_TO_SPELL
-    eval $returnvar=\"$spell\"
 }
 
 #---------------------------------------------------------------------
@@ -1013,6 +1177,87 @@ function private_up_trigger() {
 }
 
 #---------------------------------------------------------------------
+## @param spell providing the sub-depends
+## @param name of sub-depends
+##
+## Request a sub-depends from $SPELL on $1 for $2
+## This queues the sub-depends for later processing, if the sub-dependee
+## does not already support the sub-depends.
+#---------------------------------------------------------------------
+function real_sub_depends() {
+  $STD_DEBUG
+  local requester=$SPELL
+  local sub_dependee=$1
+  local sub_depends=$2
+
+  # add to hashes for lookup later in depends resolution
+  hash_append sub_dep_f_hash $requester $sub_dependee:$sub_depends $'\n' &&
+  hash_append sub_dep_r_hash $sub_dependee $requester:$sub_depends $'\n'
+
+  message "${SPELL_COLOR}${SPELL}${DEFAULT_COLOR}" \
+          "${CHECK_COLOR}requests" \
+          "${SPELL_COLOR}$sub_dependee${DEFAULT_COLOR}${CHECK_COLOR}" \
+          "with ${SPELL_COLOR}$sub_depends${DEFAULT_COLOR}"
+  if spell_ok $sub_dependee &&
+     [[ $(search_sub_depends "$SUB_DEPENDS_STATUS" "$requester" "$sub_dependee" "$sub_depends") != "" ]]; then
+    debug "libdepends" "$FUNCNAME -- $requester -> $sub_dependee -> $sub_depends provided by installed spell"
+  else
+    if ! echo "$PRETEND_NOT_INSTALLED" | grep -q " $1 "  &&
+       spell_held $sub_dependee; then
+      # sub-depends isnt in already installed spell, and spell is held
+      # (and not specified on cli) so we cant recast it.
+      # Fail here as early in the process as possible
+      message "${PROBLEM_COLOR}Sub-depends requested on a held spell${DEFAULT_COLOR}"
+      return 1
+    fi
+    if spell_installed "$sub_dependee"; then
+      # FIXME, could check tablet and do more clever things here
+      if [[ $(search_sub_depends "$SUB_DEPENDS_STATUS" "" "$sub_dependee" "$sub_depends") != "" ]]; then
+        debug "libdepends" "$sub_dependee has a $sub_depends from another spell"
+        add_sub_depends "$SUB_DEPENDS_STATUS" "$requester" \
+                        "$sub_dependee" "$sub_depends"
+        return 0
+      fi
+      # the spell is installed, as a last ditch effort, run the
+      # "PRE_SUB_DEPENDS" file, to see if it provides the sub-depends but
+      # just doesn't have it listed
+      (
+        THIS_SUB_DEPENDS=$sub_depends
+        if ! tablet_set_spell $sub_dependee ; then
+          codex_set_current_spell_by_name $sub_dependee || return 1
+        fi
+        test -x $SCRIPT_DIRECTORY/PRE_SUB_DEPENDS || return 1
+        persistent_load &&
+        source $SCRIPT_DIRECTORY/PRE_SUB_DEPENDS; rc=$?
+        persistent_save
+        return $rc
+      ) && {
+        debug "libdepends" "adding altruistic sub-depends for $sub_dependee with $sub_depends from $requester"
+        add_sub_depends "$SUB_DEPENDS_STATUS" "$requester" "$sub_dependee" "$sub_depends"
+        return 0
+      }
+    fi
+    # spell is either not installed, or is installed but without the
+    # sub-depends, we must (re)cast it
+    NEW_SUB_DEPENDEES=( ${NEW_SUB_DEPENDEES[*]} $sub_dependee )
+    hash_put sub_depends_process $sub_dependee "yes"
+  fi
+  return 0
+}
+
+#---------------------------------------------------------------------
+## Allow a spell to altruisticly specify that it provides a certain
+## sub-depends.
+## @param sub-depends being provided altruistically
+#---------------------------------------------------------------------
+function real_altruistic_sub_depends() {
+  local sub_depends=$1
+  local sub_depends_file rsub_depends_file
+  get_uncommitted_sub_depends_file "$SPELL" sub_depends_file || return 1
+  add_sub_depends "$sub_depends_file" "" "$SPELL" "$sub_depends"
+}
+
+#---------------------------------------------------------------------
 ## all the depends callbacks eventually bottom out here
 ## if a spell depends or doesnt depend on some other spell
 ## @param Spell
@@ -1098,7 +1343,7 @@ function private_add_triggerees() {
 function private_add_depends()
 {
   debug "libdepends" "$FUNCNAME: SPELL=$SPELL, NEW_DEPENDS=${NEW_DEPENDS[*]}"
-  hash_put "$CAST_HASH" "$SPELL" "${NEW_DEPENDS[*]}"
+  hash_append "$CAST_HASH" "$SPELL" "${NEW_DEPENDS[*]}"
 
   for child in ${NEW_DEPENDS[*]}; do
     hash_append "$BACK_CAST_HASH" "$child" "$SPELL"
@@ -1112,7 +1357,7 @@ function private_add_depends()
     hash_append "$BACK_CAST_HASH" "basesystem" "$SPELL"
   fi
 
-  spells=( ${spells[*]} ${NEW_DEPENDS[*]} )
+  spells=( ${spells[*]} ${NEW_DEPENDS[*]} ${NEW_SUB_DEPENDEES[*]} )
 
   TRIGGEREES=( ${TRIGGEREES[*]} ${triggerees[*]} )
   spells=( ${spells[*]} ${triggerees[*]} )
@@ -1121,6 +1366,27 @@ function private_add_depends()
 
 }
 
+#---------------------------------------------------------------------
+## Determine if spell should be lazily updated, possible asks the user
+## what to do.
+#---------------------------------------------------------------------
+function want_lazy_update() {
+  if [[ $LAZY_DEPENDS_UPDATES ]] && [[ $LAZY_DEPENDS_UPDATES != ignore ]] ; then
+    if does_spell_need_update "$1" ; then
+      message "${SPELL_COLOR}$1${DEFAULT_COLOR}${CHECK_COLOR}" \
+              "needs updating${DEFAULT_COLOR}"
+      if [[ "$LAZY_DEPENDS_UPDATES" == always ]] ; then
+        return 0
+      else
+        default=n
+        [[ $LAZY_DEPENDS_UPDATES == ask-yes ]] && default=y
+        query "Would you like to update it?" $default && return 0 || return 1
+      fi
+    fi
+  fi
+  return 1
+}
+
 
 #########################BEGIN OTHER STUFF############################
 
diff --git a/var/lib/sorcery/modules/libdepengine b/var/lib/sorcery/modules/libdepengine
index 845f92d..a43662f 100644
--- a/var/lib/sorcery/modules/libdepengine
+++ b/var/lib/sorcery/modules/libdepengine
@@ -7,12 +7,12 @@
 ## In the simple model, without triggers, we color graph nodes (spells)
 ## white, and as we recursiely visit them, mark them grey. When all of
 ## a spell's children have been visited, and successfully cast, we
-## cast the spell, and mark the node either black:0 or black:1 for success
-## or failure respectively. If a child fails we mark the current node
-## black:1 and return. In the case that we visit a child that is grey,
-## a dependency loop is detected, currently we ignore it and build the
-## tail spell anyway, in the future we could break optional depends and
-## cast some spell twice.
+## cast the spell, and mark the node either black:0 or
+## black:<non-zero> (eg black:34) for success or failure respectively.
+## If a child fails we mark the current node black:<non-zero> and
+## return. In the case that we visit a child that is grey, a dependency
+## loop is detected, currently we ignore it and build the tail spell anyway,
+## in the future we could break optional depends and cast some spell twice.
 ##
 ## The more complex model implemented below includes the above, but after
 ## a spell builds (or fails), it colors itself brown and executes its triggers.
@@ -107,19 +107,18 @@ function depengine_cast_engine() {
     # grey, this is a cycle involving triggers (yuck!)
     # when that happens dont execute $spell's triggers
     # because it will be re-triggered
-    local grey_trigerer=0
-    for trigerer in $(hash_get trg_r_hash cast_self:$spell); do
+    local grey_triggerer=0
+    for triggerer in $(hash_get trg_r_hash $spell:cast_self); do
       # note that $spell's status is grey
-      [[ $trigerer == $spell ]] && continue
-      child_status=$(hash_get state_hash $trigerer)
+      [[ $triggerer == $spell ]] && continue
+      child_status=$(hash_get state_hash $triggerer)
       if [[ $child_status == grey ]] ; then
-         #[[ $child_status == brown ]] && [[ $org_color == brown ]] ; then
-        debug "libdepengine" "$spell has a $child_status trigerer on $trigerer!"
-        grey_trigerer=1
+        debug "libdepengine" "$spell has a $child_status triggerer on $triggerer!"
+        grey_triggerer=1
         break
       fi
     done
-    if [[ $grey_trigerer == 0 ]] ; then
+    if [[ $grey_triggerer == 0 ]] ; then
       execute_triggers $spell 0
     fi
   else
@@ -138,7 +137,7 @@ function depengine_cast_engine() {
 ## @param Spell
 ## @param In trigger flag (optional) if 1, then back off more readily in
 ## the event of a dependency loop.
-## @global MINUS_K if set, then act like make -k and continue building
+## @global MINUS_K if 'yes', then act like make -k and continue building
 ## dependent spells even if another one fails.
 ##
 #------------------------------------------------------------------------
@@ -153,12 +152,17 @@ function recurse_depends() {
     # check if any of our dependencies failed while we weren't looking
     # if one did, bail out, if MINUS_K is set then skip this and build
     # everything we can
-    if ! [[ $MINUS_K ]] ; then
+    if [[ $MINUS_K == no ]] ; then
       for _child in $(hash_get dep_f_hash $spell); do
-        if [[ $(dpgn_get_spell_color $_child) == black:1 ]] ; then
-          debug libdepengine "$spell already had a failed dep on $_child"
-          return 1
-        fi
+        case $(dpgn_get_spell_color $_child) in
+          black:0)
+            :
+          ;;
+          black:*)
+            debug libdepengine "$spell already had a failed dep on $_child"
+            return 1
+          ;;
+        esac
       done
     fi
 
@@ -171,7 +175,7 @@ function recurse_depends() {
           tmp_rc=$?
           if [[ $tmp_rc != 0 ]] ; then
             rc=$tmp_rc
-            [[ $MINUS_K ]] || break
+            [[ $MINUS_K == no ]] && break
           fi
         fi
       ;;
@@ -189,24 +193,27 @@ function recurse_depends() {
       black:0)
         debug "libdepengine" "$child already built properly, continuing"
       ;;
-      black:1)
+      black:*)
+        debug "libdepengine" "$child failed, minus_k is $MINUS_K"
         rc=1
-        [[ $MINUS_K ]] || break
+        [[ $MINUS_K == no ]] && break
       ;;
     esac
   done
-  # build triggering spells, this helps keep the order
-  # its true however that most of the time, we'll also depend on the
-  # triggerer, this just catches those cases where we dont, but doesn't
-  # worry too much about a failure
-  for triggerer in $(hash_get trg_r_hash cast_self:$spell); do
-    trgrr_status=$(dpgn_get_spell_color $child)
-    if [[ "$trgrr_status" == "white" ]] ; then
-      if list_find "$need_cast_list" $child; then
-        depengine_cast_engine $child $in_trigger
+  # build anything that will trigger us. this helps keep the order right.
+  # most of the time, we'll also depend on the triggerer, this just
+  # catches those cases where we dont, and doesn't worry too much
+  # about a failure
+  if [[ $rc == 0 ]] ; then
+    for triggerer in $(hash_get trg_r_hash $spell:cast_self); do
+      trgrr_status=$(dpgn_get_spell_color $triggerer)
+      if [[ "$trgrr_status" == "white" ]] ; then
+        if list_find "$need_cast_list" $triggerer; then
+          depengine_cast_engine $triggerer $in_trigger
+        fi
       fi
-    fi
-  done
+    done
+  fi
   return $rc
 }
 
diff --git a/var/lib/sorcery/modules/libdispel b/var/lib/sorcery/modules/libdispel
index 509bfe5..b0a6535 100644
--- a/var/lib/sorcery/modules/libdispel
+++ b/var/lib/sorcery/modules/libdispel
@@ -41,7 +41,9 @@ function reap_config_files()  {
   local FILE
   while  read  FILE;  do
     [[ $FILE ]] || continue
-    if grep -q "$(md5sum "$FILE" )" "$MD5S"; then
+    if [[ $TOTAL_DISPEL ]] ; then
+      rm -f "$FILE"
+    elif grep -q "$(md5sum "$FILE" )" "$MD5S"; then
       debug libdispel "$FILE is a config file that hasnt changed, removing..."
       rm -f "$FILE"
     else
@@ -86,6 +88,22 @@ function reap_depends()  {
   # to always run this code to remove old depends.
   if [[ $NO_REAP_DEPENDS != "on" ]] ; then
     remove_depends_status $DEPENDS_STATUS $SPELL
+
+    # remove a sub-depends when both the requester and sub-dependee are not
+    # installed
+    for triple in $(search_sub_depends "$SUB_DEPENDS_STATUS" "$SPELL" '.*'); do
+      tmp=${triple%:*}
+      sub_dependee=${tmp#*:}
+      if ! spell_ok $sub_dependee; then
+        remove_sub_depends "$SUB_DEPENDS_STATUS" "$SPELL" "$sub_dependee"
+      fi
+    done
+    for triple in $(search_sub_depends "$SUB_DEPENDS_STATUS" '.*' "$SPELL"); do
+      requester=${triple%%:*}
+      if [[ -z "$requester" ]] || ! spell_ok $requester; then
+        remove_sub_depends "$SUB_DEPENDS_STATUS" "$requester" "$SPELL"
+      fi
+    done
   fi
 }
 
@@ -115,6 +133,7 @@ function reaper()  {
   local REAPER_FILES="$TMP_DIR/reaper.$UNIQUE.files"
   local REAPER_DIRS="$TMP_DIR/reaper.$UNIQUE.dirs"
   local REAPER_SYMS="$TMP_DIR/reaper.$UNIQUE.syms"
+  local REAPER_SPECIAL="$TMP_DIR/reaper.$UNIQUE.special"
 
   rm  -f  $REAPER_FILES  $REAPER_DIRS  $REAPER_SYMS
 
@@ -129,6 +148,9 @@ function reaper()  {
     if   test -h  "$ITEM"; then echo "$ITEM" >> $REAPER_SYMS
     elif test -f  "$ITEM"; then echo "$ITEM" >> $REAPER_FILES
     elif test -d  "$ITEM"; then echo "$ITEM" >> $REAPER_DIRS
+    # if it isn't a symlink, regular file or directory, assume it is a
+    # special file (character, block, or fifo)
+    elif test -e  "$ITEM"; then echo "$ITEM" >> $REAPER_SPECIAL
     fi
   done
 
@@ -147,10 +169,11 @@ function reaper()  {
   fi
 
   [ -f $REAPER_SYMS ] && rm -f `cat $REAPER_SYMS` 2>/dev/null
+  [ -f $REAPER_SPECIAL ] && rm -f `cat $REAPER_SPECIAL` 2>/dev/null
   [ -f $REAPER_DIRS ] && rmdir `sort -r $REAPER_DIRS` 2>/dev/null
   [ -f $REAPER_FILES ] && rmdir `dirnames < $REAPER_FILES |uniq|sort  -r` 2>/dev/null
 
-  rm  -f  $REAPER_FILES  $REAPER_DIRS  $REAPER_SYMS
+  rm  -f  $REAPER_FILES  $REAPER_DIRS  $REAPER_SYMS $REAPER_SPECIAL
 }
 
 #---------------------------------------------------------------------
diff --git a/var/lib/sorcery/modules/libdownload b/var/lib/sorcery/modules/libdownload
index bad3157..a96d9e8 100644
--- a/var/lib/sorcery/modules/libdownload
+++ b/var/lib/sorcery/modules/libdownload
@@ -75,11 +75,6 @@ function dl_get_bucket() {
   local dl_target=$5
   local dl_type=$6
 
-  if [ "$handler" == "svns" ]
-  then
-  	local handler=svn
-  fi
-
   if ! [[ "$handler" ]] ; then
     message "Missing handler, this is probably a sorcery bug"
     return 255
diff --git a/var/lib/sorcery/modules/libmedia b/var/lib/sorcery/modules/libmedia
index f0cb839..abe2d9d 100644
--- a/var/lib/sorcery/modules/libmedia
+++ b/var/lib/sorcery/modules/libmedia
@@ -190,7 +190,7 @@ color_schemes_menu()  {
 
       B)  COLOR_SCHEME=blueish ;;
       D)  COLOR_SCHEME=dark    ;;
-      L)  COLOR_SCHEME=bright  ;;
+      L)  COLOR_SCHEME=light  ;;
 
     esac
     remove_config $LOCAL_CONFIG COLOR_SCHEME
diff --git a/var/lib/sorcery/modules/libmisc b/var/lib/sorcery/modules/libmisc
index 27249ff..846221a 100644
--- a/var/lib/sorcery/modules/libmisc
+++ b/var/lib/sorcery/modules/libmisc
@@ -472,7 +472,7 @@ function iterate()
 }
 
 #---------------------------------------------------------------------
-## @param return_var (must not be i, foo, temp, returnvar, or default)
+## @param return_var (must not be i, foo, temp, returnvar, stuff or default)
 ## @param default choice
 ## @param elements, ..
 ## 
@@ -486,26 +486,50 @@ function select_list()
     local foo temp number
     local returnvar=$1
     local default=$2
+    local stuff=()
+
     shift 2
+    hash_unset select_list_hash
+    # see note in select_provider
+    stuff=(0 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z)
+    
     let i=0
     for foo in "$@"; do
-        message "\t$DEFAULT_COLOR($i)  $SPELL_COLOR$foo$DEFAULT_COLOR"
-        temp[$i]="$foo"
+        message "\t$DEFAULT_COLOR(${stuff[$i]})  $SPELL_COLOR$foo$DEFAULT_COLOR"
+        hash_put select_list_hash "${stuff[$i]}" "$foo"
         let i++
     done
 
-    message -n "\n${QUERY_COLOR}Which one do you want? [$default]$DEFAULT_COLOR "
-    read   -t  $PROMPT_DELAY  -n  1  number
-    if [[ ! $number ]] ; then number=$default; fi
-    while [[ $number != [0-9]* ]] || (( $number >= $i )) ; do
-        message -n "\n${QUERY_COLOR}Which one do you want? [$default]$DEFAULT_COLOR "
-        read -n 1 number
-        if [[ ! $number ]] ; then number=$default; fi
-    done
+    local msg="\n${QUERY_COLOR}Which one do you want? [$default]$DEFAULT_COLOR "
+    select_list_sub "$returnvar" select_list_hash "$msg" "$default"
+    hash_unset select_list_hash
+}
+
+#---------------------------------------------------------------------
+## Common code for select_list and select_provider
+## The user should have already printed out the menu, this handles
+## getting a valid answer from the user.
+## @param name of return value
+## @param name of hash table mapping answers to results
+## @param message to print out for the query
+## @param default answer
+#---------------------------------------------------------------------
+function select_list_sub() {
+  local returnvar=$1
+  local hashname=$2
+  local msgstr=$3
+  local default=$4
 
-    echo
+  local result
 
-    eval $returnvar=\"${temp[$number]}\"
+  while [[ ! $result ]] ; do
+    message -n "$msgstr"
+    read   -t  $PROMPT_DELAY  -n  1  answer
+    [[ $answer ]] || answer=$default
+    result="$(hash_get $hashname $answer)"
+  done
+  echo
+  eval $returnvar=\"$result\"
 }
 
 #---------------------------------------------------------------------
@@ -1331,6 +1355,39 @@ function define_functions() {
   done
 }
 
+#---------------------------------------------------------------------
+## @param target
+## @param value
+## Set variable by name, useful for setting variables that were passed
+## by reference.
+## Example:
+##
+## function uber() {
+##   local x
+##  unter x 5
+##  echo $x
+## }
+## function unter() {
+##   local var=$1 y=$2
+##   read -p "enter value (default $2)" y
+##   upvar $var $y
+## }
+## 
+#---------------------------------------------------------------------
+function upvar() {
+  # This makes eval see things as name=$2,
+  # which makes leading/trailing whitespace and special chars magically work.
+  eval "$1=\$2"
+
+}
+
+#---------------------------------------------------------------------
+## Like uniq, but doesnt require a sorted list, implemented in awk.
+#---------------------------------------------------------------------
+function awkuniq() {
+  awk '{ if ( seen[$0] != 1) { seen[$0]=1;print $0;}}'
+}
+
 # Standard debug line:
 # file "function@line" "all" "args"
 STD_DEBUG='eval local _stddbg_file=${BASH_SOURCE[0]} ; 
diff --git a/var/lib/sorcery/modules/libsorcery b/var/lib/sorcery/modules/libsorcery
index 857b867..a121773 100644
--- a/var/lib/sorcery/modules/libsorcery
+++ b/var/lib/sorcery/modules/libsorcery
@@ -290,15 +290,6 @@ function save_libraries()  {
   else  export  LD_LIBRARY_PATH="$OLD_LIBS:$LD_LIBRARY_PATH"
   fi
 
-  # NOTE: remove this in 1.13
-  if  [  "$SPELL"  ==  "glibc"         ]   ||
-      [  "$SPELL"  ==  "glibc-custom"  ] ;  then
-    if [ -e $OLD_LIBS/ld-$OLD_VERSION.so ] ; then 
-      [[ `uname -m` == ppc* ]] && ln  -sf  $OLD_LIBS/ld-$OLD_VERSION.so  ${INSTALL_ROOT}/lib/ld.so.1 ||
-      ln  -sf  $OLD_LIBS/ld-$OLD_VERSION.so  ${INSTALL_ROOT}/lib/ld-linux.so.2 
-    fi
-  fi
-
 }
 
 
@@ -494,10 +485,12 @@ function filter_generic() {
   local filter_name=$2
   local system_filter=$3
   local grimoire_filter section_filter spell_filter spell_dir
-  tablet_find_spell_dir $SPELL spell_dir &>/dev/null &&
-  tablet_get_spell_filter $spell_dir $filter_name spell_filter &>/dev/null
-  tablet_get_section_filter $spell_dir $filter_name section_filter &>/dev/null
-  tablet_get_grimoire_filter $spell_dir $filter_name grimoire_filter &>/dev/null
+  if tablet_find_spell_dir $SPELL spell_dir &>/dev/null; then
+    debug libsorcery "filter_generic tablet at : $spell_dir "
+    tablet_get_spell_filter "$spell_dir" "$filter_name" spell_filter &>/dev/null
+    tablet_get_section_filter "$spell_dir" "$filter_name" section_filter &>/dev/null
+    tablet_get_grimoire_filter "$spell_dir" "$filter_name" grimoire_filter &>/dev/null
+  fi
   set -- $args
   if [ "$1" == -v ] ; then
     shift
@@ -705,9 +698,9 @@ function optimize()  {
       esac
     done
     # use echo to crunch all the whitespace out for broken configure scripts
-    export CFLAGS=$(echo -O $CFLAGS $CUSTOM_CFLAGS $OVERRIDE_CFLAGS)
-    export CXXFLAGS=$(echo $CFLAGS $CUSTOM_CXXFLAGS $OVERRIDE_CXXFLAGS)
-    export LDFLAGS=$(echo $LDFLAGS $CUSTOM_LDFLAGS $OVERRIDE_LDLAGS)
+    export CFLAGS=$(echo $CFLAGS $CUSTOM_CFLAGS $OVERRIDE_CFLAGS $USER_SPELL_CFLAGS)
+    export CXXFLAGS=$(echo $CFLAGS $CUSTOM_CXXFLAGS $OVERRIDE_CXXFLAGS $USER_SPELL_CXXFLAGS)
+    export LDFLAGS=$(echo $LDFLAGS $CUSTOM_LDFLAGS $OVERRIDE_LDLAGS $USER_SPELL_LDFLAGS)
   fi
   export LC_ALL="C"
 
@@ -743,6 +736,56 @@ function list_install_queue() {
   fi
 }
 
+#---------------------------------------------------------------------
+## Frontend routine to encapsulate whether or not a spell needs updating
+## 
+#---------------------------------------------------------------------
+function does_spell_need_update() {
+  local line curr_updated curr_version status
+  local spell=$1
+  # this is in a subshell because the caller may not want to actually load
+  # the spell yet
+  (
+    line=$(search_spell_status $SPELL_STATUS $spell)
+    codex_set_current_spell_by_name  $spell
+    explode "$line" ":" "info"
+    does_spell_need_update_sub "$spell" "${info[3]}" "${info[1]}" /dev/null
+  )
+}
+
+#---------------------------------------------------------------------
+##
+## common code between update_install_queue and does_spell_need_update
+## This helps keep update_install_queue optimized.
+#---------------------------------------------------------------------
+function does_spell_need_update_sub() {
+  local spell=$1
+  local curr_version=$2
+  local curr_updated=$3
+  local tmp_queue=$4
+  if  [[ $VERSION != $curr_version ]] ; then
+    echo $spell >> $tmp_queue
+    return 0
+  else
+    curr_patchlevel=0
+    curr_sec_patch=0
+    if tablet_find_spell_dir $spell page_dir ; then
+      curr_updated=0
+      tablet_get_updated $page_dir curr_updated
+      tablet_get_patchlevel $page_dir curr_patchlevel
+      tablet_get_security_patch $page_dir curr_sec_patch
+    fi
+    if (( "${PATCHLEVEL:-0}" >  "$curr_patchlevel" )) ||
+       # the following is subject to change
+       (( "${SECURITY_PATCH:-0}" >  "$curr_sec_patch" )) ||
+       (( "${UPDATED:-0}" >  "${curr_updated:-0}" )) ; then
+      echo $spell >> $tmp_queue
+      return 0
+    fi
+  fi
+  return 1
+}
+
 
 #---------------------------------------------------------------------
 ##
@@ -769,24 +812,8 @@ function update_install_queue()  {
       # still want queueing to work
       if  [[  "$status"  ==  "installed"  ]] &&
         codex_set_current_spell_by_name  $spell;  then
-        if  [[ $VERSION != $curr_version ]] ; then
-          echo $spell >> $tmp_queue
-        else
-          curr_patchlevel=0
-          curr_sec_patch=0
-          if tablet_find_spell_dir $spell page_dir ; then
-            curr_updated=0
-            tablet_get_updated $page_dir curr_updated
-            tablet_get_patchlevel $page_dir curr_patchlevel
-            tablet_get_security_patch $page_dir curr_sec_patch
-          fi
-          if (( "${PATCHLEVEL:-0}" >  "$curr_patchlevel" )) ||
-             # the following is subject to change
-             (( "${SECURITY_PATCH:-0}" >  "$curr_sec_patch" )) ||
-             (( "${UPDATED:-0}" >  "${curr_updated:-0}" )) ; then
-            echo $spell >> $tmp_queue
-          fi
-        fi
+        does_spell_need_update_sub "$spell" "$curr_version" \
+                                   "$curr_updated" "$tmp_queue"
       fi
     )
   done
diff --git a/var/lib/sorcery/modules/libstate b/var/lib/sorcery/modules/libstate
index 2f40b1e..3990b4b 100644
--- a/var/lib/sorcery/modules/libstate
+++ b/var/lib/sorcery/modules/libstate
@@ -9,7 +9,7 @@
 
 #---------------------------------------------------------------------
 ##
-## Adds a dependency to the a depends database file.  Returns 1 is the 
+## Adds a dependency to the a depends database file.  Returns 1 is the
 ## 3rd or 4th fields are not valid.
 ##
 ## @param  Depends status file
@@ -25,56 +25,29 @@
 #---------------------------------------------------------------------
 function add_depends()
 { # $1=depends file $2=spell, $3=depends, $4=on/off, $5=optional/required, $6=on arg, $7=off arg
-	debug "libstate" "add_depends() - $*"
+  debug "libstate" "add_depends() - $*"
 
-	#already here for some reason
-	search_depends_status_exact "$@" >/dev/null  && return 0
-	
-        #probably not needed with commit on succeed
-	#remove stale depends info
-	search_depends_status "$@" >/dev/null && remove_depends_status "$@"
+  #already here for some reason
+  search_depends_status_exact "$@" >/dev/null  && return 0
 
-        local depends_status=$1
-        shift
-	
-	#ensure the info is valid (perhaps add check that spells exist?)
-	if ( [[ $3 != on ]]       && [[ $3 != off ]]       )   ||
-           ( [[ $4 != required ]] && [[ $4 != optional ]]  )
-	then
-	  return 1 
-	fi
-	
-	echo "$1:$2:$3:$4:$5:$6" >> $depends_status
-	
-	return 0	
+  local depends_status=$1
+  shift
 
-}
+  # Allow add_depends to override previous ones for the same pair of spells
+  search_depends_status "$depends_status" "$1" "$2" >/dev/null &&
+  remove_depends_status "$depends_status" "$1" "$2"
+
+
+  #ensure the info is valid (perhaps add check that spells exist?)
+  if ( [[ $3 != on ]]       && [[ $3 != off ]]      ) ||
+     ( [[ $4 != required ]] && [[ $4 != optional ]] ); then
+    return 1
+  fi
+
+  echo "$1:$2:$3:$4:$5:$6" >> $depends_status
+
+  return 0
 
-#---------------------------------------------------------------------
-## Sets up the uncommitted depends file. If the name isn't
-## found in the hash table create it. If the file already exists
-## move it to the abandoned depends directory and start with a new one.
-## The uncommitted_hash hash table should be hash_export'ed to
-## make it through the call to make.
-##
-## @param spell
-## @param variable name to put the filename name in (pass by reference)
-##
-#---------------------------------------------------------------------
-function get_uncommitted_depends_file() {
-    local SPELL=$1
-    local temp_spell_depends=$(hash_get "uncommitted_hash" $SPELL)
-    if ! [[ $temp_spell_depends ]] ; then
-      temp_spell_depends="$UNCOMMITTED_DEPENDS/$SPELL"
-      hash_put "uncommitted_hash" "$SPELL" "$temp_spell_depends"
-    fi
-    eval "$2=\"$temp_spell_depends\""
-    if [ -e $temp_spell_depends ]; then
-      mkdir -p "$ABANDONED_DEPENDS"
-      mv "$temp_spell_depends" "$ABANDONED_DEPENDS"
-    fi
-    mkdir -p "$UNCOMMITTED_DEPENDS"
-    touch "$temp_spell_depends"
 }
 
 #---------------------------------------------------------------------
@@ -97,13 +70,46 @@ function get_uncommitted_depends_file() {
 function search_depends_status_exact()
 { # $1=depends file $2=spell, $3=depends, $4=on/off, $5=optional/required, $6=on arg, $7=off arg
 
-        local depends_status=$1
-        shift
+  local depends_status=$1
+  shift
+
+  local a1 a2 a3 a4 a5 a6
+  a1=`esc_str "$1"` ; a2=`esc_str "$2"` ; a3=`esc_str "$3"` ; a4=`esc_str "$4"`; a5=`esc_str "$5"`; a6=`esc_str "$6"`
+  debug "libstate" "search_depends_status_exact: [$a1:$a2:$a3:$a4:$a5:$a6]"
+  grep "^$a1:$a2:$a3:$a4:$a5:$a6$" $depends_status
+
+}
+
+#---------------------------------------------------------------------
+##
+## Search depends status, dont do anything escaping.
+##
+## Arguments can be regexp
+##
+## Prints out the matching line(s)
+## @Stdout the matching line(s)
+## @param depends file
+## @param spell
+## @param depends
+## @param on/off (optional)
+## @param optional/required (optional)
+## @param on arg (optional)
+## @param off arg (optional)
+##
+#---------------------------------------------------------------------
+function search_depends_status_simple()
+{ # $1=depends file $2=spell, $3=depends, $4=on/off, $5=optional/required, $6=on arg, $7=off arg
+
+    local depends_status=$1
+    shift
+
+    local a1 a2 a3 a4 a5 a6
+    a1=`esc_str "$1"` ; a2=`esc_str "$2"`
+    a3=${3:-.*} ; a4=${4:-.*}
+    a5=`esc_str "$5"`; a6=`esc_str "$6"`
+    a5=${5:-.*} ; a6=${6:-.*}
 
-	local a1 a2 a3 a4 a5 a6
-	a1=`esc_str "$1"` ; a2=`esc_str "$2"` ; a3=`esc_str "$3"` ; a4=`esc_str "$4"`; a5=`esc_str "$5"`; a6=`esc_str "$6"`
-        debug "libstate" "search_depends_status_exact: [$a1:$a2:$a3:$a4:$a5:$a6]"
-	grep "^$a1:$a2:$a3:$a4:$a5:$a6$" $depends_status
+    grep "^$a1:$a2\(([^:]*)\)\?:$a3:$a4:$a5:$a6$" $depends_status
 
 }
 
@@ -134,7 +140,7 @@ function search_depends_status()
 
 
 #---------------------------------------------------------------------
-## 
+##
 ## @param depends file
 ## @param spell
 ## @param dependency spell
@@ -181,28 +187,154 @@ function get_depends_options()
 function remove_depends_status()
 { # $1=depends file $2=spell, $3=(OPT)depends, $4=(OPT)on/off, $5=(OPT)optional/required, $6=(OPT)on arg, $7=(OPT)off arg
 
-        local depends_status=$1
-        shift
+   local depends_status=$1
+   shift
+
+    local a1 a2 a3 a4 a5 a6
+    # arguments 1 and 2 are spell names, which might have characters that
+    # look like regexps, 5 and 6 might like funny too
+    # contrast 3 and 4 which are "on", "off", "required" or "optional"
+    # if they are regexps its probably desired so dont escape those
+    a1=`esc_str "$1"`; a2=`esc_str "$2"`
+    a1=${a1:-.*}; a2=${a2:-.*}
+
+    a3=${3:-.*} ; a4=${4:-.*}
+
+    a5=`esc_str "$5"`; a6=`esc_str "$6"`
+    a5=${a5:-.*}; a6=${a6:-.*}
+    [[ "$a5" == "[none]" ]] && a5="\\[none\\]"
+
+    sedit "/^$a1:$a2:$a3:$a4:$a5:$a6$/D" $depends_status
+
+}
+
+#---------------------------------------------------------------------
+## Sets up the uncommitted depends file. If the name isn't
+## found in the hash table create it. If the file already exists
+## move it to the abandoned depends directory and start with a new one.
+## The uncommitted_hash hash table should be hash_export'ed to
+## make it through the call to make.
+##
+## @param spell
+## @param variable name to put the filename name in (pass by reference)
+##
+#---------------------------------------------------------------------
+function get_uncommitted_depends_file() {
+    local SPELL=$1
+    local temp_spell_depends=$(hash_get "uncommitted_hash" $SPELL)
+    if ! [[ $temp_spell_depends ]] ; then
+      temp_spell_depends="$UNCOMMITTED_DEPENDS/$SPELL"
+      hash_put "uncommitted_hash" "$SPELL" "$temp_spell_depends"
+
+      mkdir -p "$ABANDONED_DEPENDS"
+      mv "$temp_spell_depends" "$ABANDONED_DEPENDS" &>/dev/null
+      mv "$temp_spell_depends:*" "$ABANDONED_DEPENDS" &>/dev/null
+
+      mkdir -p "$UNCOMMITTED_DEPENDS"
+      touch "$temp_spell_depends"
+    fi
+    eval "$2=\"$temp_spell_depends\""
+}
+
+#---------------------------------------------------------------------
+## Get the uncommitted sub-depends file. Uses get_uncommitted_depends_file.
+## This file is intended for use by the sub-dependee.
+##
+## @param spell
+## @param variable name to put the filename in
+#---------------------------------------------------------------------
+function get_uncommitted_sub_depends_file() {
+  local SPELL=$1
+  local upvar=$2
+  local __tmp
+  get_uncommitted_depends_file "$SPELL" __tmp
+  # : cannot be in spell names, so we can take advantage of the namespace
+  __tmp=${__tmp}:s
+  # we can assume that the parent directory exists because
+  # get_uncommitted_depends_file creates it
+  touch "$__tmp"
+  eval "$2=\"$__tmp\""
+}
+
+#---------------------------------------------------------------------
+## Get the uncommitted sub-depends file. Uses get_uncommitted_depends_file
+## This file is intended for use by the requester.
+##
+## @param spell
+## @param variable name to put the filename in
+#---------------------------------------------------------------------
+function get_uncommitted_rsub_depends_file() {
+  local SPELL=$1
+  local upvar=$2
+  local __tmp
+  get_uncommitted_depends_file "$SPELL" __tmp
+  # : cannot be in spell names, so we can take advantage of the namespace
+  __tmp=${__tmp}:rs
+  # we can assume that the parent directory exists because
+  # get_uncommitted_depends_file creates it
+  touch "$__tmp"
+  eval "$2=\"$__tmp\""
+}
+
+#---------------------------------------------------------------------
+## Add a sub-dependency.
+##
+## @param sub-depends file
+## @param requester file
+## @param sub-dependee
+## @param sub-depends
+#---------------------------------------------------------------------
+function add_sub_depends() {
+  $STD_DEBUG
+  remove_sub_depends "$@" || return 1
+  local file=$1
+  shift
+  # FIXME LOCK
+  echo "$1:$2:$3" >> "$file"
+}
 
-	local a1 a2 a3 a4 a5 a6
-        # why do we esc_str the args? spells shouldnt have regexps in their name
-        # and "on" "off" "optional" and "required" are the only valid
-        # values for a3 and a4, also, with the exception of dispel depends
-        # this isnt ever called with more than a1 as a parameter...
-	#a1=`esc_str $1` ; a2=`esc_str $2` ; a3=`esc_str $3` ; a4=`esc_str $4`;
-	a1=$1 ; a2=$2 ; a3=$3 ; a4=$4;
-        a5=`esc_str $5`; a6=`esc_str $6`
-	
-	a2=${a2:-.*}
-	a3=${a3:-.*}
-	a4=${a4:-.*}
-	a5=${a5:-.*}
-	a6=${a6:-.*}
+#---------------------------------------------------------------------
+## Search in sub-depends file.
+##
+## @param sub-depends file
+## @param requester file (optional)
+## @param sub-dependee (optional)
+## @param sub-depends (optional)
+#---------------------------------------------------------------------
+function search_sub_depends() {
+  $STD_DEBUG
+  local file=$1
+  shift
+  local a1 a2 a3
+  a1=`esc_str "$1"` ; a2=`esc_str "$2"` ; a3=`esc_str "$3"`
+  a1=${a1:-.*}
+  a2=${a2:-.*}
+  a3=${a3:-.*}
+  # FIXME LOCK
+  grep "^$a1:$a2:$a3$" "$file"
+}
+
+#---------------------------------------------------------------------
+## Remove a sub-depends.
+##
+## @param sub-depends file
+## @param requester file
+## @param sub-dependee
+## @param sub-depends
+#---------------------------------------------------------------------
+function remove_sub_depends() {
+  $STD_DEBUG
+  local file=$1
+  shift
 
-	[[ "$a5" == "[none]" ]] && a5="\\[none\\]"
+  test -f $file || return
 
-	sedit "/^$a1:$a2:$a3:$a4:$a5:$a6$/D" $depends_status
+  local a1 a2 a3
+  a1=`esc_str "$1"`; a2=`esc_str "$2"`; a3=`esc_str "$3"`
+  a1=${a1:-.*}; a2=${a2:-.*}; a3=${a3:-.*}
 
+  # FIXME LOCK
+  sedit "/^$a1:$a2:$a3/D" "$file"
 }
 
 #---------------------------------------------------------------------
@@ -219,7 +351,7 @@ function remove_depends_status()
 # default provider examples:
 # use xorg as X11-LIBS even if its optional
 # xorg:X11-LIBS:on
-# 
+#
 # use xfree86 as X11-LIBS unless the [none] option exists (and use that instead)
 # xfree86:X11-LIBS:off
 #---------------------------------------------------------------------
@@ -263,7 +395,7 @@ function add_default_depends() {
         remove_default_depends $tfile $1 $2
         echo "$1:$2:$3" >> $tfile
         lock_commit_transaction $file
-       
+
        return 0
 
 }
@@ -285,12 +417,12 @@ function remove_default_depends() {
         shift
         test -e $file || return
 
-	local a1 a2 a3
-	a1=`esc_str $1` ; a2=`esc_str $2` ; a3=`esc_str $3`
-	
-	a3=${a3:-.*}
+        local a1 a2 a3
+        a1=`esc_str $1` ; a2=`esc_str $2` ; a3=`esc_str $3`
+
+        a3=${a3:-.*}
 
-	sedit "/^$a1:$a2:$a3$/D" $file
+        sedit "/^$a1:$a2:$a3$/D" $file
 }
 
 #---------------------------------------------------------------------
@@ -309,12 +441,12 @@ function search_default_depends() {
         shift
         [[ -s $default_depends ]] || return
 
-	local a1 a2 a3
-	a1=`esc_str "$1"` ; a2=`esc_str "$2"` ; a3=`esc_str "$3"`
-	a3=${a3:-.*}
+        local a1 a2 a3
+        a1=`esc_str "$1"` ; a2=`esc_str "$2"` ; a3=`esc_str "$3"`
+        a3=${a3:-.*}
 
         debug "libstate" "search_default_depends: [$a1:$a2:$a3]"
-	grep "^$a1:$a2:$a3$" $default_depends
+        grep "^$a1:$a2:$a3$" $default_depends
 }
 
 
@@ -329,40 +461,41 @@ function search_default_depends() {
 ##
 #---------------------------------------------------------------------
 function add_default_provider() {
-	debug "libstate" "add_default_depends() - $*"
+  debug "libstate" "add_default_depends() - $*"
 
-	#already here
-	search_default_provider "$@" >/dev/null  && return 0
+  #already here
+  search_default_provider "$@" >/dev/null  && return 0
 
-        local file=$1
-        shift
+  local file=$1
+  shift
 
-	#ensure the info is valid...
-        { [[ ! $1 ]] && [[ ! $2 ]] ; } && return 1
-        { [[ $3 != on ]] && [[ $3 != off ]] ; } && return 1
-        if [[ $1 ]] && ! codex_does_spell_exist $1 &>/dev/null ; then
-          return 1
-        fi
-        if [[ $2 ]] && ! codex_does_service_exist $2 &>/dev/null ; then
-          return 1
-        fi
+  #ensure the info is valid...
+  { [[ ! $1 ]] && [[ ! $2 ]] ; } && return 1
+  { [[ $3 != on ]] && [[ $3 != off ]] ; } && return 1
 
-        tfile=`lock_start_transaction $file`
-        remove_default_provider $tfile "" $2
-	echo "$1:$2:$3" >> $tfile
-        lock_commit_transaction $file
-	
-	return 0	
+  if [[ $1 ]] && ! codex_does_spell_exist $1 &>/dev/null ; then
+    return 1
+  fi
+  if [[ $2 ]] && ! codex_does_service_exist $2 &>/dev/null ; then
+    return 1
+  fi
+
+  tfile=`lock_start_transaction $file`
+  remove_default_provider $tfile "" $2
+  echo "$1:$2:$3" >> $tfile
+  lock_commit_transaction $file
+
+  return 0
 }
 
 # these are the same, as their countparts but are here for completeness
 # and incase they do someday diverge
 function remove_default_provider() {
-        remove_default_depends "$@"
+  remove_default_depends "$@"
 }
 
 function search_default_provider() {
-        search_default_depends "$@"
+  search_default_depends "$@"
 }
 
 #---------------------------------------------------------------------
@@ -410,6 +543,23 @@ function add_spell()  {
   add_spell_status $1 $2 $3
 }
 
+#---------------------------------------------------------------------
+## @param spell status file
+## @param spell
+## @param version (optional)
+## @Stdout mathing lines
+##
+## Arguments can be regexp
+##
+## Prints out the matching line(s)
+##
+#---------------------------------------------------------------------
+function search_spell_status() {
+  local spell_status=$1
+  shift
+  [ $# -eq 1 ] && grep "^`esc_str "$1"`:" $spell_status
+  [ $# -gt 1 ] && grep "^`esc_str "$1"`:`esc_str "$2"`:" $spell_status
+}
 
 #---------------------------------------------------------------------
 ##
@@ -422,18 +572,18 @@ function add_spell()  {
 ## a version.
 ##
 ## Prints out the matching spell's status
-## Returns 1 only for critical problems ( missing SPELL_STAUS file ) 
+## Returns 1 only for critical problems ( missing SPELL_STAUS file )
 ##         0 for all other cases ( even when spell is not found )
 ##
 #---------------------------------------------------------------------
 function query_spell_status()
 { #$1=spell, $2=(OPT)version
 
-	local version
-	version=`esc_str ${2:-".*"}`
+  local version
+  version=`esc_str ${2:-".*"}`
 
-	tac $SPELL_STATUS | grep -m 1 "^$1:.*:.*:$version$" | \
-		cut -d ":" -f 3
+  tac $SPELL_STATUS | grep -m 1 "^$1:.*:.*:$version$" | \
+  cut -d ":" -f 3
 }
 
 #---------------------------------------------------------------------
@@ -464,7 +614,7 @@ function real_spell_held()      {
 ## @return 0 if the given spell's status is "installed" or "held"
 #---------------------------------------------------------------------
 function real_spell_ok() {
-  local res="$(  query_spell_status  $1  )"
+  ''local res="$(  query_spell_status  $1  )"
   [  "$res"  ==  "installed"  ] || [  "$res"  ==  "held" ]
 }
 
@@ -492,6 +642,7 @@ function real_provider_ok() {
 ## db doesnt exist (maybe we're not casting) use DEPENDS_STATUS
 ##
 ## @return 0 if a provider could be found 1 if not
+## @return 0 if a dependency from $1 on $2 is enabled, 1 otherwise.
 ## @stdout the provider name(s)
 #---------------------------------------------------------------------
 function real_get_spell_provider() {
@@ -506,6 +657,28 @@ function real_get_spell_provider() {
                  'on' '.*'  '.*' '.*' |cut -f2 -d:|cut -f1 -d\(|sort|uniq
 }
 
+#---------------------------------------------------------------------
+## @Type API
+## @param Spell name
+## @param Target spell name
+## @param If empty get the uncommited spell info, if anything else get
+## info from the committed ($DEPENDS_STATUS) database. If the uncommited
+## db doesnt exist (maybe we're not casting) use DEPENDS_STATUS
+##
+## @return 0 if a dependency from $1 on $2 is enabled, 1 otherwise.
+## @stdout the provider name(s)
+#---------------------------------------------------------------------
+function real_is_depends_enabled() {
+  local dep_status
+  if [[ $3 ]] ; then
+    dep_status=$DEPENDS_STATUS
+  else
+    dep_status=$(hash_get uncommitted_hash $1)
+    [[ $dep_status ]] || dep_status=$DEPENDS_STATUS
+  fi
+  [[ -n $(search_depends_status_simple $dep_status "$1" "$2" 'on' ) ]]
+}
+
 
 #---------------------------------------------------------------------
 ## @param spell name
@@ -529,7 +702,7 @@ function remove_spell_status()
   tSPELL_STATUS=`lock_start_transaction $SPELL_STATUS`
   grep  -v  "^$1:" $SPELL_STATUS  >  $tSPELL_STATUS
   lock_commit_transaction $SPELL_STATUS
-    
+
 }
 
 #---------------------------------------------------------------------
@@ -672,7 +845,7 @@ function get_all_spells_with_status () {
 ## @param command (optional)
 ##
 ## Modifies (or adds) an entry in the local/config.
-## If "command" is the third argument, a space will separate the 
+## If "command" is the third argument, a space will separate the
 ## variable and value rather than the equals sign.
 ##
 #---------------------------------------------------------------------
@@ -689,7 +862,7 @@ function modify_local_config() {
 ## @param command (optional)
 ##
 ## Modifies (or adds) an entry in the local/config.
-## If "command" is the third argument, a space will separate the 
+## If "command" is the third argument, a space will separate the
 ## variable and value rather than the equals sign.
 ##
 #---------------------------------------------------------------------
@@ -698,17 +871,17 @@ function modify_config()  {
   debug "libstate" "modify_config() - $*"
   local FILE=$1
   shift;
-  
+
   if ! [[ $1 ]]; then
     debug "libstate" "modify_config() - Warning: No name given for config option."
     return 1
   fi
-  
+
   local TEMP separator EQUALS_COL VARIABLE
-  
+
   # what to use as separator?
-  if [[ $3 == command ]] 
-  then separator=" " 
+  if [[ $3 == command ]]
+  then separator=" "
   else separator="=" ; fi
 
   if ! test -f $FILE; then
@@ -736,13 +909,13 @@ function modify_config()  {
   for (( ; EQUALS_COL>0 ; EQUALS_COL-- )) ; do
     TEMP="$TEMP "
   done
-  
+
   debug "libstate" "modify_config() - entering new value $VARIABLE $seperator $@ into $tFILE"
   # put new value and justification in
   echo  "${TEMP}${1}${separator}\"$2\""  >>  $tFILE
 
   lock_commit_transaction $FILE
-    
+
 }
 
 #---------------------------------------------------------------------
@@ -751,7 +924,7 @@ function modify_config()  {
 ## @param command (optional)
 ##
 ## Modifies (or adds) an entry in the local/config.
-## If "command" is the third argument, a space will separate the 
+## If "command" is the third argument, a space will separate the
 ## variable and value rather than the equals sign.
 ##
 #---------------------------------------------------------------------
@@ -760,24 +933,24 @@ function remove_config()  {
   debug "libstate" "remove_config() - $*"
   local FILE=$1
   shift;
-  
+
   if ! [[ $1 ]]; then
     debug "libstate" "remove_config() - Warning: No name given for config option."
     return 1
   fi
-  
+
   local TEMP separator EQUALS_COL VARIABLE
-  
+
   # what to use as separator?
-  if [[ $2 == command ]] 
-  then separator=" " 
+  if [[ $2 == command ]]
+  then separator=" "
   else separator="=" ; fi
-  
+
   # remove previous reference
   tFILE=`lock_start_transaction $FILE`
   grep  -v  "^[[:blank:]]*${1}${separator}" $FILE >  $tFILE
   lock_commit_transaction $FILE
-    
+
 }
 
 #---------------------------------------------------------------------
@@ -786,16 +959,16 @@ function remove_config()  {
 ## returns true if the grimoire is set local, false otherwise.
 #---------------------------------------------------------------------
 function codex_is_local () {(
-    local g_name=$1
+  local g_name=$1
 
-    local grimoire=$(codex_find_grimoire $g_name)
-    if ! [[ $grimoire ]]; then
-        return 1
-    fi
+  local grimoire=$(codex_find_grimoire $g_name)
+  if ! [[ $grimoire ]]; then
+    return 1
+  fi
 
-    . "$grimoire/GRIMOIRE"
+  . "$grimoire/GRIMOIRE"
 
-    [[ $CODEX_IS_LOCAL == "yes" ]]
+  [[ $CODEX_IS_LOCAL == "yes" ]]
 )}
 
 
diff --git a/var/lib/sorcery/modules/libsummon b/var/lib/sorcery/modules/libsummon
index e6d5125..99d27a7 100644
--- a/var/lib/sorcery/modules/libsummon
+++ b/var/lib/sorcery/modules/libsummon
@@ -124,8 +124,11 @@ function real_acquire_src() {
   if file_exists "$SOURCE_CACHE/$target " &&
      ! [[ ${FORCE_DOWNLOAD} ]] &&
      ! [[ ${FORCE_DOWNLOAD[${DLNUM:-1}]} ]] ; then
+    # file_exists does a fuzzy match with gz, tgz and bz2
+    # lookup what file it actually found and use that for the message
+    local real_target=$(guess_filename $SOURCE_CACHE/$target)
     message  "${MESSAGE_COLOR}Found source file"             \
-             "${FILE_COLOR}${target}${DEFAULT_COLOR}"     \
+             "${FILE_COLOR}${real_target}${DEFAULT_COLOR}"     \
              "$REASON in ${FILE_COLOR}${SOURCE_CACHE}${DEFAULT_COLOR}"
     return 0
   fi
@@ -382,7 +385,7 @@ function unpack_for_update() {
   local _guess_type
 
   # hard-coded list of url prefixes that generally download trees
-  local tree_prefixes="cvs dir rsync smgl_tla svn"
+  local tree_prefixes="cvs dir rsync smgl_tla svn svn_http svn_ssh"
   local prefix=$(url_get_prefix $url_list)
   if ! list_find "$hints" file &&
      list_find "$hints" tree || list_find "$tree_prefixes" "$prefix" ; then
diff --git a/var/lib/sorcery/modules/libtablet b/var/lib/sorcery/modules/libtablet
index 36b8908..ac60940 100644
--- a/var/lib/sorcery/modules/libtablet
+++ b/var/lib/sorcery/modules/libtablet
@@ -63,6 +63,8 @@
 ##        tablet_get_patchlevel
 ##        tablet_get_security_patch
 ##        tablet_get_depends
+##        tablet_get_sub_depends
+##        tablet_get_rsub_depends
 ##        tablet_get_status
 ##        tablet_get_sources
 ##        tablet_get_spell_filter
@@ -163,6 +165,29 @@ function tablet_get_depends() {
   return 0
 }
 
+function tablet_get_sub_depends() {
+  local tb_dir=$1 tb_version value
+  tablet_get_tb_version $tb_dir tb_version
+  [[ $? != 0 ]] && return 1
+  case $tb_version in
+    1) test -f $tb_dir/sub_depends && value=$tb_dir/sub_depends|| return 2 ;;
+    *) return 3 ;;
+  esac
+  eval "$2=\"$value\""
+  return 0
+}
+
+function tablet_get_rsub_depends() {
+  local tb_dir=$1 tb_version value
+  tablet_get_tb_version $tb_dir tb_version
+  [[ $? != 0 ]] && return 1
+  case $tb_version in
+    1) test -f $tb_dir/rsub_depends && value=$tb_dir/rsub_depends|| return 2 ;;
+    *) return 3 ;;
+  esac
+  eval "$2=\"$value\""
+  return 0
+}
 function tablet_get_status() {
   local tb_dir=$1 tb_version value
   tablet_get_tb_version $tb_dir tb_version
@@ -342,8 +367,22 @@ function tablet_get_path() {
 # @param spell name
 # @param upvar
 # @param timestamp (optional)
+#
+# @global TABLET_IGNORE if set means that there isnt a tablet for this
+# and somewhere up the stack knows this
+# @global TABLET_SPELL_DIR if set, is the value of the tablet for this
+# spell, because somewhere up the stack it was just created (so dont bother
+# looking for it)
+#
+# the idea for the two globals is to simplify the layers in-between
+# and skip searching when it isn't necessary
 #----------------------------------------------------------------------
 function tablet_find_spell_dir() {
+  [[ $TABLET_IGNORE ]] && return 1
+  if [[ $TABLET_SPELL_DIR ]] ; then
+    eval "$2=\"$TABLET_SPELL_DIR\""
+    return 0
+  fi
   local SPELL=$1
   if ! test -d $TABLET_PATH; then
     mkdir -p $TABLET_PATH
@@ -446,8 +485,17 @@ function tablet_install_spell_files() {
   test -e $SPELL_CONFIG.p && cp ${SPELL_CONFIG}.p spell_config.p
 
   # depends info
+  get_uncommitted_depends_file $SPELL spell_depends &&
   test -e $spell_depends && cp $spell_depends depends
 
+  # sub-depends we provide
+  local sub_depends_file rsub_depends_file
+  get_uncommitted_sub_depends_file $SPELL sub_depends_file &&
+  test -e $sub_depends_file && cp $sub_depends_file sub_depends
+  # get sub-depend we request
+  get_uncommitted_rsub_depends_file $SPELL sub_depends_file &&
+  test -e $rsub_depends_file && cp $sub_depends_file rsub_depends
+
   # logs
   mkdir logs
   ln -s $INST_LOG logs/install
diff --git a/var/lib/sorcery/modules/libtrack b/var/lib/sorcery/modules/libtrack
index f794a22..4de9d3e 100644
--- a/'>var/lib/sorcery/modules/libtrack
+++ b/'>var/lib/sorcery/modules/libtrack
@@ -325,11 +325,7 @@ function log_adjuster() {
     fi
   fi
  
-  local TMP_SSF=$TMP_DIR/$RANDOM
-  while ! mkdir $TMP_SSF; do
-    TMP_SSF=$TMP_DIR/$RANDOM
-    sleep 1
-  done
+  local TMP_SSF=$(make_safe_dir)
   local TMP_DATA=$TMP_SSF/foo.data
   local TMP_MDATA=$TMP_SSF/foo.mdata
      
@@ -344,6 +340,7 @@ function log_adjuster() {
   } > $output
   rm $TMP_DATA $TMP_MDATA
   rmdir $TMP_SSF
+  return 0
 }
 
 #---------------------------------------------------------------------
@@ -362,21 +359,53 @@ function seperate_state_files() {
   local REAL_STATE_DIR=${STATE_DIRECTORY#$STATE_ROOT}
  
   # the input file is almost certainly a pipe, and things get weird
-  # since we have to grep twice, so just dump the data into a unique
-  # file, the loop below shouldn't go forever unless there's ~30,000 pipes
+  # since we have to grep twice, so just dump the data into a unique file
 
-  local FILE=$TMP_DIR/ssf.$RANDOM
-  while test -e $FILE; do
-    FILE=$TMP_DIR/ssf.$RANDOM
-  done
+  local TMP_SSF=$(make_safe_dir)
+  local FILE=$TMP_SSF/ssf
   cat $1 > $FILE
   grep -v "$REAL_LOG_DIR\|$REAL_STATE_DIR" $FILE | grep -xv '' > $2 $2
   grep    "$REAL_LOG_DIR\|$REAL_STATE_DIR" $FILE | grep -xv '' > $3
   rm $FILE
+  rmdir $TMP_SSF
   return 0
 } }
  
 #---------------------------------------------------------------------
+## Try to make a unique directory using $RANDOM, leverage the fact that
+## two simultaneous mkdir's will have one succeed and the other fail.
+##
+## This is run from log_adjuster primarily which may be invoked several
+## times in a pipe, when this happens, bash does a fork, but does not
+## seem to reseed the random number generator, causing a high rate of
+## collisions. This is not easily reproducable outside of sorcery at
+## the time of writing, but inside it happens nearly everytime with bash
+## 3.1. This may actually be a bash 3.1 bug.
+##
+## The collisions aren't bad necessarily, but they result in un-necesary
+## delay.
+##
+## Setting RANDOM re-seeds the random number generator.
+##+##
+## Despite the fact that the subshell and invocation of date are slow
+## the consequence for not doing them are worse. The nano-seconds
+## are usually going to be different between forks so the liklihood
+## of a collision is greatly reduced.
+#---------------------------------------------------------------------
+function make_safe_dir() { (
+  RANDOM=$(date "+%N")
+  local TMP_SSF=$TMP_DIR/$RANDOM
+  while ! mkdir "$TMP_SSF" &> /dev/null; do
+    RANDOM=$(date "+%N")
+    TMP_SSF=$TMP_DIR/$RANDOM
+    sleep .1
+    debug "libtrack" "safe dir collision on $TMP_SSF"
+  done
+  echo $TMP_SSF
+  return 0
+) }
++
+#---------------------------------------------------------------------
 ## @License
 ##
 ## This software is free software; you can redistribute it and/or modify
diff --git a/var/lib/sorcery/modules/libunpack b/ b/var/lib/sorcery/modules/libunpack
index 8742e3f..806ec3e 100644
--- a/var/lib/sorcery/modules/libunpack
+++ b/var/lib/sorcery/modules/libunpack
@@ -207,8 +207,8 @@ function unpack_core() {
  
   case  "$2"  in
             bzip2|gzip|compress*|tar)
-                    tar   --owner=root  --group=root  -xf  /dev/stdin \
-                          2> /dev/null  ||  cat > /dev/null            ;;
+                    tar   --owner=root  --group=root --no-same-permissions -xf \
+		          /dev/stdin 2> /dev/null  ||  cat > /dev/null ;;
               Zip)  cat /dev/stdin >/dev/null   #get rid of unused output
                     unzip  -q  "$1"                                    ;;
               RPM)  cpio  -idm < /dev/stdin                            ;;
diff --git a/var/lib/sorcery/modules/liburl b/var/lib/sorcery/modules/liburl
index d4df1ea..36c4979 100644
--- a/'>var/lib/sorcery/modules/liburl
+++ b/var/lib/sorcery/modules/liburl
@@ -115,12 +115,18 @@ function url_download_expand_sort() {
   # you can't expect me to download something without urls...
   [ -z "$url_list" ] && return 255
  
+  # ensure list is newline seperated
+  url_list="$(echo $url_list|tr ' ' '\n')"
++
   # expand urls
   url_expand_urls expanded_urls "$url_list"
   if ! [[ "$expanded_urls" ]] ; then
     message "No expanded urls! If you get this it is a sorcery bug"
     return 1
   fi
+  # limit the number of expanded urls, only the linux kernel reaches this
+  # limit and doing netselect on that many urls is absurd
+  expanded_urls=$(echo "$expanded_urls"|head -n 50)
   debug "liburl" "expanded urls $expanded_urls"
   # sort urls 
   url_sort_urls sorted_urls "$expanded_urls"
@@ -233,15 +239,12 @@ function url_expand_urls() {
   local upvar="$1"
   local url_list="$2"
   local tmp_list
-  tmp_list="$(for url in $url_list; do
+  # put the original urls in the front of the list
+  # awkuniq will remove duplicates thereafter
+  tmp_list="$({ echo "$url_list" ; for url in $url_list; do
     # this should expand to something or return itself
     url_expand "$url"
-  done|sort|uniq)"
-  # this ensures that the urls originally specified are given preference
-  for url in $url_list; do
-    tmp_list=$(echo "$tmp_list"|grep -v $url)
-  done
-  tmp_list="$(echo $url_list|tr ' ' '\n')"$'\n'"$tmp_list"
+  done ; } |awkuniq)"
   eval "$upvar=\"\$tmp_list\""
 }
  
@@ -282,7 +285,7 @@ function url_rank() {
  
   debug "liburl" "unsorted list: $*"
 
-  urlList="$*"
+  urlList="$@"
 
   # Even if theres one url it might have multiple A records and we'll
   # want netselect to find the fastest one
@@ -310,18 +313,22 @@ function url_rank() {
     # url_speed[69]="${url_speed[69]} url1"
     # url_speed[234]="${url_speed[234]} url2"
     eval $(url_netselect $tmp_list|
-      awk '{printf "url_speed[%s]=\"${url_speed[%s]} %s\";",$1,$1,$2}')
+      awk '{ if ($2 > 0 && $2 < 65536 ) {
+               printf "url_speed[%s]=\"${url_speed[%s]} %s\";",$1,$1,$2
+             }
+           }')
   done
   
   # since we put things in url_speed indexed by speed, the * should
   # expand back in sorted order
 
-  sortedList=$(echo ${url_speed[*]})
+  sortedList=$(echo ${url_speed[@]})
   debug "liburl" "Ordered list pre-sanity check is $sortedList"
 
   # if all sites are ICMP Unreachable, return the unsorted list instead of null.
   if [[ -z "$sortedList" ]] ; then
-    echo $urlList
+    debug "liburl" "Failed to expand list"
+    echo "$urlList"
     return
   fi
 
@@ -335,10 +342,10 @@ function url_rank() {
   # just in case something failed along the way just return what we had
   if [[ -z "$sortedList" ]] ; then
     debug "liburl" "Ordering failed somewhere, giving back original input"
-    echo $urlList
+    echo "$urlList"
   else
     debug "liburl" "Ordered URLs: $sortedList"
-    echo $sortedList
+    echo "$sortedList"
   fi
 }
 
@@ -401,7 +408,7 @@ function url_bucketize()  {
 #---------------------------------------------------------------------
 function url_crack()  {
   $STD_DEBUG
-  url_generic_apifunc crack $1
+  url_generic_apifunc crack "$@"
 }
 
 #---------------------------------------------------------------------
diff --git a/var/lib/sorcery/modules/url_handlers/url_svn b/var/lib/sorcery/modules/url_handlers/url_svn
index e79d22f..bf7c75f 100644
--- a/var/lib/sorcery/modules/url_handlers/url_svn
+++ b/var/lib/sorcery/modules/url_handlers/url_svn
@@ -3,39 +3,34 @@
 ##
 ##=head1 SYNOPSIS
 ## 
-## Url handler functions for grabbing svn urls.(subversion)
-##
-##=head1 DESCRIPTION
-##
-## This file contains functions for parsing svn urls.
+## Url handler functions for parsing subversion urls.
 ##
 ##=head1 SVN URL Format
 ##
-## There did not used to be a standard for svn urls, a source mage
-## specific format was invented:
+## The source mage specific format is:
 ##
 ##      svn://SVNURL:DIR_NAME
+##      svn://SVNURL:DIR_NAME:REVISION_TAG
 ##
-## The above url will download the latest version of the specified
-## module (i.e., the HEAD revision). To specify a specific revision,
-## the following format can be used:
+## It is exactly the same as a standard svn:// url, with additional
+## tokens at the end DIR_NAME and optional REVISITION_TAG
 ##
-##      svn://SVNURL:DIR_NAME:REVISION_TAG
+## The svn://SVNURL portion of the url will appear on the svn command
+## line as the url.
 ##
-## The SVNURL portion of the url will appear as a normal http url.
 ## The DIR_NAME will be the tail element of the SOURCE_DIRECTORY.
 ##
-## Future work will differntiate between real svn urls and our smgl
-## specific version, and expanding our smgl version to use protocols
-## besides http.
+## In order to maintain compatibility with the original svn url format,
+## which uses http:// as the underlying protocol, a hint may be specified
+## named "old_svn_compat".
 ##
 ## For more details, see the SVN manual at 
 ## http://svnbook.red-bean.com/svnbook/ch03s04.html
 ##
 ##=head1 EXAMPLES
 ##
-## Suppose we want to download the latest version of the sorcery
-## scripts from svn.  We'd use the following url:
+## Suppose we want to download the latest version of bmp-plugins
+## from svn.  We'd use the following url:
 ##
 ##      svn://svn.pld-linux.org/svn/bmp-plugins/trunk:bmp-plugins-svn
 ##
@@ -43,15 +38,12 @@
 ##
 ##      svn://svn.pld-linux.org/svn/bmp-plugins/trunk:bmp-plugins-svn:4474
 ##
-## The assumption is made that the svn url is actually an http url.
-## ie. you could use your web browser to access:
-## http://svn.pld-linux.org/svn/bmp-plugins/trunk
-##
 ## svn repositories requiring passwords are not currently supported.
 ##
 ##=head1 COPYRIGHT
 ##
 ## Copyright 2004 by the Source Mage Team
+## Copyright 2005 by the Source Mage Team
 ##
 ##=head1 FUNCTIONS
 ##
@@ -60,7 +52,7 @@
 #---------------------------------------------------------------------
 
 #---------------------------------------------------------------------
-##=item url_file_download <url>
+##=item url_svn_crack <url>
 ## 
 ## Parse the specified svn url.
 ##
@@ -68,18 +60,17 @@
 ## @Global SVN_ROOT
 ## @Global SVN_MODULE
 ## @Global SVN_TAG
-## @Global SVN_TYPE
+##
 #---------------------------------------------------------------------
 function url_svn_crack() { 
 
-  SVN_TYPE=`echo $1 | cut -d ':' -f 1`
-  if [ "$SVN_TYPE" == "svns" ]
-  then
-  	URL=`url_strip_prefix "$1" svns`
+  URL=`url_strip_prefix "$1" svn`
+  SVN_ROOT=`echo $URL | sed "s#\(^[^/]*[^:]*\):.*#\1#"`
+  if list_find "$2" old_svn_compat; then
+    SVN_ROOT=http://$SVN_ROOT
   else
-  	URL=`url_strip_prefix "$1" svn`
+    SVN_ROOT=svn://$SVN_ROOT
   fi
-  SVN_ROOT=`echo $URL | sed "s#\(^[^/]*[^:]*\):.*#\1#"`
   local SVN_MODULE_TAG=`echo $URL | sed "s#^[^/]*[^:]*\(.*\)#\1#"`
   SVN_MODULE=`echo $SVN_MODULE_TAG | cut -d : -f2` 
   local SVN_TAGNAME=`echo $SVN_MODULE_TAG | cut -d : -f3`