[SM-Commit] BZR Change 20 to devel sorcery by Andrew Stitt <astitt at sourcemage.org>

bzr at mail.sourcemage.org bzr at mail.sourcemage.org
Fri Apr 14 12:15:05 EDT 2006


------------------------------------------------------------
revno: 20
committer: Andrew Stitt <astitt at sourcemage.org>
branch nick: devel
timestamp: Fri 2006-04-14 09:11:56 -0700
message:
  fix bug 9386 -- optimize tablet_import_repair_files
    ------------------------------------------------------------
    merged: astitt at sourcemage.org-20060414070549-73f8d2b8f9a06a26
    committer: Andrew Stitt <astitt at sourcemage.org>
    branch nick: devel-hack3
    timestamp: Fri 2006-04-14 00:05:49 -0700
    message:
      Optimize tablet_import_repair_files, now runs about 10 times faster.
      Completes in about 6 seconds for me, compared with over 60 seconds.
      
      * Removed all but one fork (from codex in-memory cache creation).
      * Replace codex_set_current_spell_by_name with codex_get_spell_paths and
        hash_get_ref on the in-memory cache.
      * Move tablet lookups for version, updated, and patchlevel to
        tablet_check_repair_file, so they are only run when actually needed.
      * replace find -type f -mindepth 1 -maxdepth 1 .... with glob pattern
        plus a [[ -f ]]
      * use smgl_basename and smgl_dirname where possible
      * replace $(cat) with $(<)
      
      Also had to update cleanse routines since they tie into the repair file routines
    ------------------------------------------------------------
    merged: astitt at sourcemage.org-20060414061038-ccd58caf381a28c4
    committer: Andrew Stitt <astitt at sourcemage.org>
    branch nick: devel-hack3
    timestamp: Thu 2006-04-13 23:10:38 -0700
    message:
      add function to pass up hash values by reference instead of using stdout
    ------------------------------------------------------------
    merged: astitt at sourcemage.org-20060414060954-9cd051f486683914
    committer: Andrew Stitt <astitt at sourcemage.org>
    branch nick: devel-hack3
    timestamp: Thu 2006-04-13 23:09:54 -0700
    message:
      improve in-memory cache creation by providing an option to only store
      info for spells that are actually installed (easy 10x speedup
      for typical installs)
    ------------------------------------------------------------
    merged: astitt at sourcemage.org-20060414043631-fe729b362d0cfd7f
    committer: Andrew Stitt <astitt at sourcemage.org>
    branch nick: devel-hack3
    timestamp: Thu 2006-04-13 21:36:31 -0700
    message:
      add routines to create in-memory spell lookup table
    ------------------------------------------------------------
    merged: astitt at sourcemage.org-20060414043417-6913968692c52e4d
    committer: Andrew Stitt <astitt at sourcemage.org>
    branch nick: devel-hack3
    timestamp: Thu 2006-04-13 21:34:17 -0700
    message:
      move common code for setting spell paths into a seperate function
      this is one of many things necessary to make tablet_import_repair_files as
      fast as possible
    ------------------------------------------------------------
    merged: astitt at sourcemage.org-20060414041654-c932672e93e14ae0
    committer: Andrew Stitt <astitt at sourcemage.org>
    branch nick: devel-hack3
    timestamp: Thu 2006-04-13 21:16:54 -0700
    message:
      add smgl_basename and smgl_dirname functions

=== modified file 'ChangeLog'
--- ChangeLog	
+++ ChangeLog	
@@ -1,3 +1,13 @@
+2006-04-14 Andrew Stitt <astitt at sourcemage.org>
+	* libmisc: add forkless basename and dirname functions using
+	  bash variable expansions.
+	* libhash: add lookup routine that returns by reference
+	* libcodex: add routines to setup in-memory cache lookup hash, and
+	  factor out code to setup spell paths from spell loading functions
+	* libtablet: fix bug 9386, optimize tablet_import_repair_files,
+	  also tie up loose ends with cleanse related functions
+	* cleanse: update parameters to tablet_cleanse_chapter
+	
 2006-04-13 Andrew Stitt <astitt at sourcemage.org>
 	* libsorcery: consolidate history functions, bug 10282
 	  modified patch from Jaka Kranjc. Also added a long overdue full

=== modified file 'usr/sbin/cleanse'
--- usr/sbin/cleanse	
+++ usr/sbin/cleanse	
@@ -159,7 +159,7 @@
         fi
         ;;
       --tablet_spell)
-        tablet_cleanse_chapter $TABLET_PATH/$1 $BACKUPDIR
+        tablet_cleanse_chapter $TABLET_PATH/$1 $BACKUPDIR yes
         let rc+=$?
         shift
         ;;

=== modified file 'var/lib/sorcery/modules/libcodex'
--- var/lib/sorcery/modules/libcodex	
+++ var/lib/sorcery/modules/libcodex	
@@ -596,32 +596,11 @@
 function codex_set_current_spell()  {
 
   debug "libcodex" "runing codex_set_current_spell"
+
   codex_clear_current_spell
-  SPELL_DIRECTORY=$1
-
-  # Directories
-  SCRIPT_DIRECTORY=$SPELL_DIRECTORY
-  # scribbled spells are self contained, SECTION_DIRECTORY and GRIMOIRE
-  # point to slightly different places
-  if test -e $SPELL_DIRECTORY/SCRIBBLED ; then
-    # I dont know how to nest these
-    local tmp=${SPELL_DIRECTORY%/*}
-    SECTION=${tmp##*/}
-    GRIMOIRE_NAME=$(basename ${tmp%/*})
-
-    SECTION_DIRECTORY=${SPELL_DIRECTORY}/section
-    GRIMOIRE=${SPELL_DIRECTORY}/grimoire
-  else
-    SECTION_DIRECTORY=${SPELL_DIRECTORY%/*}
-    SECTION=${SECTION_DIRECTORY##*/}
-    GRIMOIRE=${SECTION_DIRECTORY%/*}
-    GRIMOIRE_NAME=$(basename $GRIMOIRE)
-  fi
-
-  # Names
-  SPELL=${SPELL_DIRECTORY##*/}
+
+  codex_get_spell_paths $1
   
-  SPELL_CONFIG="$DEPENDS_CONFIG/$SPELL"
   if  [ -f  $SPELL_CONFIG  ]; then
     .  $SPELL_CONFIG > /dev/null  2> /dev/null
   fi
@@ -645,33 +624,46 @@
   true
 }
 
+#---------------------------------------------------------------------
+# Load just the very basics for a spell, skip build api, libcompat
+# and other debug stuff. Intended for use in tight loops.
+#---------------------------------------------------------------------
 function codex_set_current_spell_quick()  {
-  SPELL_DIRECTORY=$1
-  SCRIPT_DIRECTORY=$1
-  if test -e $SPELL_DIRECTORY/SCRIBBLED ; then
-    # I dont know how to nest these
-    local tmp=${SPELL_DIRECTORY%/*}
-    SECTION=${tmp##*/}
-    GRIMOIRE_NAME=$(basename ${tmp%/*})
-
-    SECTION_DIRECTORY=${SPELL_DIRECTORY}/section
-    GRIMOIRE=${SPELL_DIRECTORY}/grimoire
-  else
-    SECTION_DIRECTORY=${SPELL_DIRECTORY%/*}
-    SECTION=${SECTION_DIRECTORY##*/}
-    GRIMOIRE=${SECTION_DIRECTORY%/*}
-    GRIMOIRE_NAME=$(basename $GRIMOIRE)
-  fi
-
-  # Names
-  SPELL=${SPELL_DIRECTORY##*/}
-  SPELL=${1##*/}
-  SPELL_CONFIG="$DEPENDS_CONFIG/$SPELL"
+  codex_get_spell_paths $1
+
   test -f  "$SPELL_CONFIG" && .  "$SPELL_CONFIG" &> /dev/null
   persistent_load
   .  $1/DETAILS &> /dev/null
   persistent_clear
   true
+}
+
+
+#---------------------------------------------------------------------
+# Setup the various spell paths associated with a spell without
+# loading it.
+#---------------------------------------------------------------------
+function codex_get_spell_paths() {
+  SPELL_DIRECTORY=$1
+  SCRIPT_DIRECTORY=$1
+  if test -e $SPELL_DIRECTORY/SCRIBBLED ; then
+    local tmp
+    smgl_dirname "$SPELL_DIRECTORY" tmp
+    smgl_basename "$tmp" SECTION
+    smgl_dirname "$tmp" tmp
+    smgl_basename "$tmp" GRIMOIRE_NAME
+
+    SECTION_DIRECTORY=${SPELL_DIRECTORY}/section
+    GRIMOIRE=${SPELL_DIRECTORY}/grimoire
+  else
+    smgl_dirname "$SPELL_DIRECTORY" SECTION_DIRECTORY
+    smgl_basename "$SECTION_DIRECTORY" SECTION
+    smgl_dirname "$SECTION_DIRECTORY" GRIMOIRE
+    smgl_basename "$GRIMOIRE" GRIMOIRE_NAME
+  fi
+  smgl_basename "$SPELL_DIRECTORY" SPELL
+  SPELL_CONFIG="$DEPENDS_CONFIG/$SPELL"
+  return 0
 }
 
 #---------------------------------------------------------------------
@@ -758,6 +750,50 @@
 
 }
 
+
+#---------------------------------------------------------------------
+## Create in-memory spell lookup table for named grimoires.
+#---------------------------------------------------------------------
+function codex_create_in_memory_cache() {
+  local list
+  if [[ $1 == -i ]] ; then
+    list=$SPELL_STATUS
+    shift
+  fi
+  local hash=$1
+  shift
+  eval $(
+    for each in "$@"; do
+      cat $each/$SPELL_INDEX_FILE
+    done | awk -v i=$list -v hash=$hash '{
+      if ( ! ($1 in map)) {
+        map[$1]=$2;
+      }
+    } END {
+      if (i) {
+        FS=":";
+        while (getline < i) {
+          if( $3=="installed" || $3=="held") {
+            printf("hash_put %s %s \"%s/%s\";\n",hash,$1,map[$1],$1);
+          }
+        }
+      } else {
+        for (spell in map) {
+          printf("hash_put %s %s \"%s/%s\";\n",hash,spell,map[spell],spell);
+        }
+      }
+    }'
+  )
+
+}
+
+#---------------------------------------------------------------------
+## Creae in-memory spell lookup table for all known grimoires.
+#---------------------------------------------------------------------
+function codex_create_in_memory_cache_all() {
+  codex_create_in_memory_cache "$@" $(codex_get_all_grimoires)
+}
+
 ###########################MISC FUNCTIONS#############################
 
 #---------------------------------------------------------------------

=== modified file 'var/lib/sorcery/modules/libhash'
--- var/lib/sorcery/modules/libhash	
+++ var/lib/sorcery/modules/libhash	
@@ -148,6 +148,23 @@
 #---------------------------------------------------------------------
 ## @param table name
 ## @param field name
+## @param upvar name
+## 
+## @Stdout none
+## Returns the value stored in the table/field through the upvar variable
+## name. If no value was previously stored in the table/field, then an
+## empty string is returned.
+##
+#---------------------------------------------------------------------
+function hash_get_ref() {
+  local VARIABLE_NAME
+  hash_build_variable_name "$1" "$2" VARIABLE_NAME
+  eval "$3=\${!VARIABLE_NAME}"
+}
+
+#---------------------------------------------------------------------
+## @param table name
+## @param field name
 ## @param value
 ## 
 ## Appends the value to the specified table/field.

=== modified file 'var/lib/sorcery/modules/libmisc'
--- var/lib/sorcery/modules/libmisc	
+++ var/lib/sorcery/modules/libmisc	
@@ -1387,6 +1387,69 @@
   awk '{ if ( seen[$0] != 1) { seen[$0]=1;print $0;}}'
 }
 
+#---------------------------------------------------------------------
+# Dirname written in bash, optionally uses an upvar (making it forkless).
+#---------------------------------------------------------------------
+function smgl_dirname() {
+  local __dir=$1
+  local __last_dir __next_dir
+
+  # Pull of any trailing /'s, there could be more than/one///
+  while [[ "$__dir" != "$__last_dir" ]] ; do
+    __last_dir=$__dir
+    __dir=${__dir%/}
+  done
+
+  # remove the basename
+  __next_dir=${__dir%/*}
+
+  # if the string had no more directory seperators in it (was aaa/, now aaa),
+  # then this is a special case, the answer is '.'
+  if [[ "$__next_dir" == "$__dir" ]] ; then
+    __dir="."
+  else
+    __dir=$__next_dir
+    # Trim off anymore leading /'s (was /aa//bb, now /aa//)
+    while [[ "$__dir" != "$__last_dir" ]] ; do
+      __last_dir=$__dir
+      __dir=${__dir%/}
+      [[ "$__dir" == "/" ]] && break
+    done
+  fi
+
+  if [[ "$2" ]] ; then
+    upvar "$2" "$__dir"
+  else
+    echo "$__dir"
+  fi
+}
+
+#---------------------------------------------------------------------
+# basename written in bash, optionally uses upvar (making it forkless).
+# Does not have ability to remove file extension like the real basename.
+#---------------------------------------------------------------------
+function smgl_basename() {
+  local __base=$1
+  local __last_base
+
+  # trim off leading /'s (foo/bar//// -> foo/bar)
+  while [[ "$__base" != "$__last_base" ]] ; do
+    __last_base=$__base
+    __base=${__base%/}
+  done
+
+  # if there are no more /'s then this does nothing which is what we want
+  __base=${__base##*/}
+
+  if [[ "$2" ]] ; then
+    upvar "$2" "$__base"
+  else
+    echo "$__base"
+  fi
+}
+
+
+
 # Standard debug line:
 # file "function at line" "all" "args"
 STD_DEBUG='eval local _stddbg_file=${BASH_SOURCE[0]} ; 

=== modified file 'var/lib/sorcery/modules/libtablet'
--- var/lib/sorcery/modules/libtablet	
+++ var/lib/sorcery/modules/libtablet	
@@ -305,7 +305,7 @@
     return 1
   fi
   if ! test -f $tb_dir/tb_version ||
-     [[ "$(cat $tb_dir/tb_version)" == 0 ]] ; then
+     [[ "$(< $tb_dir/tb_version)" == 0 ]] ; then
     tablet_0_repair $tb_dir || return 1
   fi
   local ____tb_version=$(<$tb_dir/tb_version)
@@ -329,7 +329,10 @@
 ## basename of that is <spell>
 #---------------------------------------------------------------------
 function tablet_get_spell_name_from_path() {
-  [[ $1 ]] && basename $(dirname $1)
+  [[ $1 ]] || return
+  local tmp
+  smgl_dirname "$1" tmp
+  smgl_basename "$tmp" $2 # quotes deliberatly left off, (hack for speed)
 }
 
 ######################################################################
@@ -576,56 +579,83 @@
 
 
 ############################ REPAIR files #############################
-function tablet_import_repair_files() {
+#---------------------------------------------------------------------
+## Import repair files for all tablet pages
+#---------------------------------------------------------------------
+function tablet_import_repair_files() { (
   local tablet_path=$1
   local dir rc chapter page
+
+  codex_create_in_memory_cache_all -i spell_lookup_hash
+
+  shopt -s nullglob # so REPAIR^* evaluates to nothing if no REPAIR files
 
   for chapter in $tablet_path/* ; do
     test -d $chapter || continue
     for page in $chapter/*; do
       test -d $page || continue
-      tablet_import_repair_files_page $page
+      tablet_import_repair_files_page "$page" spell_lookup_hash
     done
   done
-}
-
-function tablet_import_repair_files_page() { (
+) }
+
+#---------------------------------------------------------------------
+## Import repair files for a specific tablet page, expects caller
+## to have spell_lookup_hash setup for a spell lookup, and nullglob set.
+#---------------------------------------------------------------------
+function tablet_import_repair_files_page() {
   local page=$1
-  local spell=$(tablet_get_spell_name_from_path $page)
-  codex_set_current_spell_by_name $spell || return 1
+
+  local spell spell_dir
+  tablet_get_spell_name_from_path "$page" spell
+  hash_get_ref $2 $spell spell_dir
+  codex_get_spell_paths "$spell_dir"
+
+  local name key
+  local loaded
   local spell_version spell_updated codex_md5 repair_file tablet_file tablet_md5
-  local name key
-  tablet_get_version $page spell_version
-  tablet_get_updated $page spell_updated
-  tablet_get_patchlevel $page spell_patchlevel
-  for repair_file in $(find $SPELL_DIRECTORY -maxdepth 1 -mindepth 1 -type f -name 'REPAIR\^*'); do
-    name=$(basename $repair_file|cut -f3- -d^)
-    key=$(basename $repair_file|cut -f2 -d^)
+  for repair_file in $SPELL_DIRECTORY/REPAIR^*; do
+    [[ -f "$repair_file" ]] || continue
+    name=$(smgl_basename "$repair_file"|cut -f3- -d^)
+    key=$(smgl_basename "$repair_file"|cut -f2 -d^)
     [[ $name ]] || continue 
     [[ $key ]] || continue 
     tablet_check_repair_file spell
   done
 
-  for repair_file in $(find $SECTION_DIRECTORY -maxdepth 1 -mindepth 1 -type f -name 'REPAIR\^*'); do
-    name=$(basename $repair_file|cut -f3- -d^)
-    key=$(basename $repair_file|cut -f2 -d^)
+  for repair_file in $SECTION_DIRECTORY/REPAIR^*; do
+    [[ -f "$repair_file" ]] || continue
+    name=$(smgl_basename "$repair_file"|cut -f3- -d^)
+    key=$(smgl_basename "$repair_file"|cut -f2 -d^)
     [[ $name ]] || continue 
     [[ $key ]] || continue 
     tablet_check_repair_file section
   done
 
-  for repair_file in $(find $GRIMOIRE -maxdepth 1 -mindepth 1 -type f -name 'REPAIR\^*'); do
-    name=$(basename $repair_file|cut -f3- -d^)
-    key=$(basename $repair_file|cut -f2 -d^)
+  for repair_file in $GRIMOIRE/REPAIR^*; do
+    [[ -f "$repair_file" ]] || continue
+    name=$(smgl_basename "$repair_file"|cut -f3- -d^)
+    key=$(smgl_basename "$repair_file"|cut -f2 -d^)
     [[ $name ]] || continue 
     [[ $key ]] || continue 
     tablet_check_repair_file grimoire
   done
-) }
-
+}
+
+#---------------------------------------------------------------------
+## Private subroutine for tablet_import_repair_files_page, do not call
+## from anywhere else.
+#---------------------------------------------------------------------
 function tablet_check_repair_file() {
   local type=$1
   local replace=0
+
+  if ! [[ "$loaded" ]] ; then
+    tablet_get_version "$page" spell_version
+    tablet_get_updated "$page" spell_updated
+    tablet_get_patchlevel "$page" spell_patchlevel
+    loaded=done
+  fi
 
   case $type in
     spell)  tablet_get_spell_file $page $name tablet_file;;
@@ -672,15 +702,16 @@
   local file_backup_dir=$backup_dir/tablet_files
   local dir rc
 
-  mkdir -p $file_backup_dir
+  mkdir -p "$file_backup_dir"
   for dir in $tablet_path/* ; do
-    if test -d $dir ; then
-      tablet_cleanse_chapter $dir $backup_dir
+    if test -d "$dir" ; then
+      tablet_cleanse_chapter "$dir" "$backup_dir" no
     else
       message "$dir is not a directory! backing it up in $backup_dir"
-      mv $dir $file_backup_dir
+      mv "$dir" "$file_backup_dir"
     fi
   done
+  tablet_import_repair_files "$tablet_path"
 }
 
 #---------------------------------------------------------------------
@@ -689,20 +720,26 @@
 function tablet_cleanse_chapter() {
   local sdir=$1
   local backup_dir=$2
+  local do_repair=$3
   local spell=$(basename $sdir)
   local page rc
 
-  mkdir -p $backup_dir/duplicates
-  mkdir -p $backup_dir/chapter_files
-  tablet_fix_duplicates $spell $sdir $backup_dir/duplicates
-  test -d $sdir || return 0
+  mkdir -p "$backup_dir/duplicates"
+  mkdir -p "$backup_dir/chapter_files"
+  tablet_fix_duplicates "$spell" "$sdir" "$backup_dir/duplicates"
+  test -d "$sdir" || return 0
+
 
   for page in $sdir/*; do
-    if ! test -d $page; then
+    if ! test -d "$page"; then
       message "$dir is not a directory! backing it up in $backup_dir"
-      mv $page $backup_dir/chapter_files/$page.$spell
-    else
-      tablet_import_repair_files_page $page
+      mv "$page" "$backup_dir/chapter_files/$page.$spell"
+    elif [[ $do_repair == yes ]] ; then
+      ( # pretend we're tablet_import_repair_files
+        shopt -s nullglob
+        hash_put spell_lookup_hash "$spell" "$(codex_find_spell_by_name $spell)"
+        tablet_import_repair_files_page "$page" spell_lookup_hash
+      )
     fi
   done
 }




More information about the SM-Commit mailing list