Source Mage code commit list

[Andra?? Levstik <scm AT mail.sourcemage.org>]

GIT changes to master quill by Andra?? Levstik <ruskie AT mages.ath.cx>:

 todo                                  |    2 
 var/lib/quill/ChangeLog               |    9 +++
 var/lib/quill/modules/libfreshmeatxml |   82 
+++++++++++++++-------------------
 var/lib/quill/modules/libperlcpan     |   12 ++++
 var/lib/quill/modules/librubyraa      |    3 -
 5 files changed, 57 insertions(+), 51 deletions(-)

New commits:
commit 0aa2e6a72ff48b963bf6ef41c478e20966f56d18
Author: Andra?? Levstik <ruskie AT mages.ath.cx>
Commit: Andra?? Levstik <ruskie AT mages.ath.cx>

    improved the downloading code in perlcpan, rubyraa and freshmeatxml,
    sanitized and improved libfreshmeatxml, added downloading of fm project
    html page as well

diff --git a/todo b/todo
index ef663ec..01a68fb 100644
--- a/todo
+++ b/todo
@@ -26,8 +26,6 @@ improvements in the standard mode:
   * don't use summon. as it should be able to be run without root privs.
 
 fmxml:
-  * sanitize, dump curl?
-    I belive there is a problem with the xml if not running curl
   * improve the speed, dump xml-grep?
     Erm planning to implement own XML parser?
 
diff --git a/var/lib/quill/ChangeLog b/var/lib/quill/ChangeLog
index 5dcf61b..3a3a1fc 100644
--- a/var/lib/quill/ChangeLog
+++ b/var/lib/quill/ChangeLog
@@ -2,8 +2,15 @@
        * quill: added ruby-raa mode
        * libcore: added ruby-raa mode
        * librubyraa: added for ruby-raa mode, error checking added in case
-         no project exist, improved it a lot with Jaka's help
+         no project exist, improved it a lot with Jaka's help,
+         less verbose
        * version: 0.2.2-2007-02-24
+       * libfreshmeatxml: removed dependency on curl, improved it and 
cleaned up
+         a bit, removed a the hunt src url function as it didn't work really 
good
+         and replaced with the raw url from the page which should be parsable
+         with the already existing url fixer(if not that needs an update),
+         made it less verbose
+       * libperlcpan: improved the downloading code, less verbose
 
 2007-02-18 Jaka Kranjc <lynxlynxlynx AT sourcemage.org>
        * quill, libcore, libhistory, libupdate: fix #13536 by using a dirty 
flag
diff --git a/var/lib/quill/modules/libfreshmeatxml 
b/var/lib/quill/modules/libfreshmeatxml
index 440e1b8..9c0f121 100644
--- a/var/lib/quill/modules/libfreshmeatxml
+++ b/var/lib/quill/modules/libfreshmeatxml
@@ -3,41 +3,37 @@
 #---
 
 function quill_fmxml_grab_from_xml() {
-  xml_grep --text_only "$1" "${QUILL_TMP_DIR}/${2}.xml"
-}
-
-
-function quill_fmxml_hunt_src_url() {
-  local URL
-  local SPELL_NAME=$1
-  URL=$(curl -I $2 2>/dev/null|grep Location|awk '{print $2}'|sed 's/\r//g')
-  # hacky attempt to get source url from sourceforge
-  if echo $URL|grep -q sourceforge; then
-    curl $URL 2>/dev/null|tr '<>' '\n\n'|grep http|grep sourceforge|\
-                              grep download|grep -i $SPELL_NAME|\
-                              cut -f2 -d\"|cut -f1 -d?|head -n 1
-  else
-    echo $URL
-  fi
+  xml_grep --text_only "${1}" "${2}"
 }
 
 function quill_fmxml_core(){
   # if the argument looks like a url then use that, otherwise
   # assume the argument is the name of the spell and construct a
   # url for where we think freshmeat.net stores their xml data
-  if echo $1|grep -q http://; then
+  if grep -q "http://"; <<< "${1}" ; then
     FRESHMEAT_XML_URL="$1"
   else
-    FRESHMEAT_XML_URL=http://freshmeat.net/projects-xml/${1}/${1}.xml
+    FRESHMEAT_XML_URL="http://freshmeat.net/projects-xml/${1}/${1}.xml";
+    FRESHMEAT_PROJECT_URL="http://freshmeat.net/projects/${1}/";
   fi
 
   # check if we already have the xml file, if we dont, download it
-  FRESHMEAT_FILE=$QUILL_TMP_DIR/$1.xml
-  if ! test -f $FRESHMEAT_FILE; then
-    if ! curl -o $FRESHMEAT_FILE $FRESHMEAT_XML_URL; then
-      error_msg "Failed to get $1"
-      return 1
-    fi
+  FRESHMEAT_FILE="${QUILL_TMP_DIR}/${1}.xml"
+  FRESHMEAT_PROJECT_FILE="${QUILL_TMP_DIR}/${1}.fm"
+  if [[ ! -e ${FRESHMEAT_FILE} ]]; then
+    message "Attempting to retrive project XML page..."
+    wget -q -Uquill -O ${FRESHMEAT_FILE} "${FRESHMEAT_XML_URL}" || \
+         (error_msg "Error: unable to fetch project XML page" && return 1)
+  fi
+
+  if [[ ! -e ${FRESHMEAT_PROJECT_FILE} ]]; then
+    message "Attempting to retrive project page..."
+    wget -q -Uquill -O ${FRESHMEAT_PROJECT_FILE} "${FRESHMEAT_PROJECT_URL}" 
|| \
+         (error_msg "Error: unable to fetch project page" && return 1)
+    # for some reason the dos2unix.sed from the sedders bag didn't work here 
:(
+    # but this does
+    cat ${FRESHMEAT_PROJECT_FILE} | tr "\r" " " > 
${FRESHMEAT_PROJECT_FILE}.tmp
+    mv ${FRESHMEAT_PROJECT_FILE}.tmp ${FRESHMEAT_PROJECT_FILE}
   fi
 
   # check that the project was found - exsists
@@ -45,32 +41,28 @@ function quill_fmxml_core(){
   if grep -q "$not_found" "$FRESHMEAT_FILE"; then
     error_msg "$not_found"
     rm "$FRESHMEAT_FILE"
-    SPELL_NAME="$1"
-    return 2
+    return 1
   fi
 
   # fill in variables from xml file
-  if [[ $1 ]] ; then
-    SPELL_NAME=$(quill_fmxml_grab_from_xml projectname_short $1|tr 'A-Z' 
'a-z')
-    SPELL_NAME="${SPELL_NAME:-$1}"
-    SPELL_DESC_NAME="${SPELL_NAME}"
-    for each in url_bz2 url_tgz; do
-      TMP=$(quill_fmxml_grab_from_xml $each $1)
-      if [[ $TMP ]] ; then
-        SPELL_SRC_URL=$(quill_fmxml_hunt_src_url $SPELL_NAME $TMP)
-        break
-      fi
-    done
-    SPELL_URL=$(curl -I $(quill_fmxml_grab_from_xml url_homepage $1) 2>&1|\
-                grep Location|awk '{print $2}'|sed 's/\r//g')
-
-    SPELL_LICENSE=$(quill_fmxml_grab_from_xml license $1)
-    if grep -q "(.*)" <<< "$SPELL_LICENSE"; then
-      SPELL_LICENSE=$(awk '{print $NF}' <<< "$SPELL_LICENSE" | tr -d '()')
+  SPELL_NAME=$(quill_fmxml_grab_from_xml projectname_short ${FRESHMEAT_FILE} 
|tr 'A-Z' 'a-z')
+  SPELL_NAME="${SPELL_NAME:-$1}"
+  SPELL_DESC_NAME="${SPELL_NAME}"
+  local tmp_url
+  for each in url_bz2 url_tgz url_zip; do
+    tmp_url="$(quill_fmxml_grab_from_xml $each ${FRESHMEAT_FILE})"
+    if [[ $tmp_url ]] ; then
+      SPELL_SRC_URL="$(grep "$each" ${FRESHMEAT_PROJECT_FILE} | grep -v 
"\<img\ src\>" | sed -e "s: *<a href.*\">\(.*\)</a><br>:\1:")"
+      break
     fi
-    SPELL_SHORT_DESCRIPTION=$(quill_fmxml_grab_from_xml desc_short $1|sed 
's/\r//g')
-    quill_fmxml_grab_from_xml desc_full $1 |fmt > 
${QUILL_TMP_DIR}/${SPELL_NAME}
+  done
+  SPELL_URL="$(grep "url_homepage" ${FRESHMEAT_PROJECT_FILE} | grep -v 
"\<img\ src\>" | sed -e "s: *<a href.*\">\(.*\)</a><br>:\1:")"
+  SPELL_LICENSE="$(quill_fmxml_grab_from_xml license ${FRESHMEAT_FILE})"
+  if grep -q "(.*)" <<< "${SPELL_LICENSE}"; then
+    SPELL_LICENSE="$(awk '{print $NF}' <<< "$SPELL_LICENSE" | tr -d '()')"
   fi
+  SPELL_SHORT_DESCRIPTION="$(quill_fmxml_grab_from_xml desc_short 
${FRESHMEAT_FILE}|sed 's/\r//g')"
+  quill_fmxml_grab_from_xml desc_full ${FRESHMEAT_FILE} |fmt > 
${QUILL_TMP_DIR}/${SPELL_NAME}
 }
 #---
 ##
diff --git a/var/lib/quill/modules/libperlcpan 
b/var/lib/quill/modules/libperlcpan
index 2f3e7e3..0d08513 100644
--- a/var/lib/quill/modules/libperlcpan
+++ b/var/lib/quill/modules/libperlcpan
@@ -8,8 +8,16 @@ function quill_get_cpan_meta()
      [ ! -f $QUILL_TMP_DIR/03modlist.data.gz ]
   then
     cd $QUILL_TMP_DIR
-    wget -O 02packages.details.txt.gz 
ftp://ftp.perl.org/pub/CPAN/modules/02packages.details.txt.gz
-    wget -O 03modlist.data.gz 
ftp://ftp.perl.org/pub/CPAN/modules/03modlist.data.gz
+    message "Attempting to retrive 02packages.details.txt.gz..."
+    wget -q \
+         -O 02packages.details.txt.gz \
+         ftp://ftp.perl.org/pub/CPAN/modules/02packages.details.txt.gz ||
+         (error_msg "Error: unable to fetch 02packages.details.txt.gz" && 
return 1)
+    message "Attempting to retrive 03modlist.data.gz..."
+    wget -q \
+         -O 03modlist.data.gz \
+         ftp://ftp.perl.org/pub/CPAN/modules/03modlist.data.gz ||
+         (error_msg "Error: unable to fetch 03modlist.data.gz" && return 1)
   fi
   eval QUILL_CPAN_INFO="( $(zgrep -w -E "^${1}" 
$QUILL_TMP_DIR/02packages.details.txt.gz) )"
   eval QUILL_CPAN_METADATA="( $(zgrep -w -B1 -A9 -E "^'${1}'" 
$QUILL_TMP_DIR/03modlist.data.gz | tr -d "[]" | sed 's:,$::g' ) )"
diff --git a/var/lib/quill/modules/librubyraa 
b/var/lib/quill/modules/librubyraa
index 3f65e08..89ef8ed 100644
--- a/var/lib/quill/modules/librubyraa
+++ b/var/lib/quill/modules/librubyraa
@@ -20,7 +20,8 @@ function quill_raa_core(){
   message "Attempting to retrive project page..."
   wget -q -O $raa_project_file.tmp \
        http://raa.ruby-lang.org/project/$project/ || \
-       error_msg "Error: unable to fetch project page"
+       error_msg "Error: unable to fetch project page" && \
+       return 1
   sed -e "s:<a href=\"\(.*\)\".*:\1:g" \
       -e ":top;/<.*>/{;s/<[^<>]*>//g;t top;};/</{;N;b top;}" \
       $raa_project_file.tmp | \