Skip to Content.
Sympa Menu

sm-commit - [SM-Commit] GIT changes to master quill by Andra?? Levstik (0aa2e6a72ff48b963bf6ef41c478e20966f56d18)

sm-commit AT lists.ibiblio.org

Subject: Source Mage code commit list

List archive

Chronological Thread  
  • From: Andra?? Levstik <scm AT mail.sourcemage.org>
  • To: sm-commit AT lists.ibiblio.org
  • Subject: [SM-Commit] GIT changes to master quill by Andra?? Levstik (0aa2e6a72ff48b963bf6ef41c478e20966f56d18)
  • Date: Sat, 24 Feb 2007 09:13:44 -0600

GIT changes to master quill by Andra?? Levstik <ruskie AT mages.ath.cx>:

todo | 2
var/lib/quill/ChangeLog | 9 +++
var/lib/quill/modules/libfreshmeatxml | 82
+++++++++++++++-------------------
var/lib/quill/modules/libperlcpan | 12 ++++
var/lib/quill/modules/librubyraa | 3 -
5 files changed, 57 insertions(+), 51 deletions(-)

New commits:
commit 0aa2e6a72ff48b963bf6ef41c478e20966f56d18
Author: Andra?? Levstik <ruskie AT mages.ath.cx>
Commit: Andra?? Levstik <ruskie AT mages.ath.cx>

improved the downloading code in perlcpan, rubyraa and freshmeatxml,
sanitized and improved libfreshmeatxml, added downloading of fm project
html page as well

diff --git a/todo b/todo
index ef663ec..01a68fb 100644
--- a/todo
+++ b/todo
@@ -26,8 +26,6 @@ improvements in the standard mode:
* don't use summon. as it should be able to be run without root privs.

fmxml:
- * sanitize, dump curl?
- I belive there is a problem with the xml if not running curl
* improve the speed, dump xml-grep?
Erm planning to implement own XML parser?

diff --git a/var/lib/quill/ChangeLog b/var/lib/quill/ChangeLog
index 5dcf61b..3a3a1fc 100644
--- a/var/lib/quill/ChangeLog
+++ b/var/lib/quill/ChangeLog
@@ -2,8 +2,15 @@
* quill: added ruby-raa mode
* libcore: added ruby-raa mode
* librubyraa: added for ruby-raa mode, error checking added in case
- no project exist, improved it a lot with Jaka's help
+ no project exist, improved it a lot with Jaka's help,
+ less verbose
* version: 0.2.2-2007-02-24
+ * libfreshmeatxml: removed dependency on curl, improved it and
cleaned up
+ a bit, removed a the hunt src url function as it didn't work really
good
+ and replaced with the raw url from the page which should be parsable
+ with the already existing url fixer(if not that needs an update),
+ made it less verbose
+ * libperlcpan: improved the downloading code, less verbose

2007-02-18 Jaka Kranjc <lynxlynxlynx AT sourcemage.org>
* quill, libcore, libhistory, libupdate: fix #13536 by using a dirty
flag
diff --git a/var/lib/quill/modules/libfreshmeatxml
b/var/lib/quill/modules/libfreshmeatxml
index 440e1b8..9c0f121 100644
--- a/var/lib/quill/modules/libfreshmeatxml
+++ b/var/lib/quill/modules/libfreshmeatxml
@@ -3,41 +3,37 @@
#---

function quill_fmxml_grab_from_xml() {
- xml_grep --text_only "$1" "${QUILL_TMP_DIR}/${2}.xml"
-}
-
-
-function quill_fmxml_hunt_src_url() {
- local URL
- local SPELL_NAME=$1
- URL=$(curl -I $2 2>/dev/null|grep Location|awk '{print $2}'|sed 's/\r//g')
- # hacky attempt to get source url from sourceforge
- if echo $URL|grep -q sourceforge; then
- curl $URL 2>/dev/null|tr '<>' '\n\n'|grep http|grep sourceforge|\
- grep download|grep -i $SPELL_NAME|\
- cut -f2 -d\"|cut -f1 -d?|head -n 1
- else
- echo $URL
- fi
+ xml_grep --text_only "${1}" "${2}"
}

function quill_fmxml_core(){
# if the argument looks like a url then use that, otherwise
# assume the argument is the name of the spell and construct a
# url for where we think freshmeat.net stores their xml data
- if echo $1|grep -q http://; then
+ if grep -q "http://"; <<< "${1}" ; then
FRESHMEAT_XML_URL="$1"
else
- FRESHMEAT_XML_URL=http://freshmeat.net/projects-xml/${1}/${1}.xml
+ FRESHMEAT_XML_URL="http://freshmeat.net/projects-xml/${1}/${1}.xml";
+ FRESHMEAT_PROJECT_URL="http://freshmeat.net/projects/${1}/";
fi

# check if we already have the xml file, if we dont, download it
- FRESHMEAT_FILE=$QUILL_TMP_DIR/$1.xml
- if ! test -f $FRESHMEAT_FILE; then
- if ! curl -o $FRESHMEAT_FILE $FRESHMEAT_XML_URL; then
- error_msg "Failed to get $1"
- return 1
- fi
+ FRESHMEAT_FILE="${QUILL_TMP_DIR}/${1}.xml"
+ FRESHMEAT_PROJECT_FILE="${QUILL_TMP_DIR}/${1}.fm"
+ if [[ ! -e ${FRESHMEAT_FILE} ]]; then
+ message "Attempting to retrive project XML page..."
+ wget -q -Uquill -O ${FRESHMEAT_FILE} "${FRESHMEAT_XML_URL}" || \
+ (error_msg "Error: unable to fetch project XML page" && return 1)
+ fi
+
+ if [[ ! -e ${FRESHMEAT_PROJECT_FILE} ]]; then
+ message "Attempting to retrive project page..."
+ wget -q -Uquill -O ${FRESHMEAT_PROJECT_FILE} "${FRESHMEAT_PROJECT_URL}"
|| \
+ (error_msg "Error: unable to fetch project page" && return 1)
+ # for some reason the dos2unix.sed from the sedders bag didn't work here
:(
+ # but this does
+ cat ${FRESHMEAT_PROJECT_FILE} | tr "\r" " " >
${FRESHMEAT_PROJECT_FILE}.tmp
+ mv ${FRESHMEAT_PROJECT_FILE}.tmp ${FRESHMEAT_PROJECT_FILE}
fi

# check that the project was found - exsists
@@ -45,32 +41,28 @@ function quill_fmxml_core(){
if grep -q "$not_found" "$FRESHMEAT_FILE"; then
error_msg "$not_found"
rm "$FRESHMEAT_FILE"
- SPELL_NAME="$1"
- return 2
+ return 1
fi

# fill in variables from xml file
- if [[ $1 ]] ; then
- SPELL_NAME=$(quill_fmxml_grab_from_xml projectname_short $1|tr 'A-Z'
'a-z')
- SPELL_NAME="${SPELL_NAME:-$1}"
- SPELL_DESC_NAME="${SPELL_NAME}"
- for each in url_bz2 url_tgz; do
- TMP=$(quill_fmxml_grab_from_xml $each $1)
- if [[ $TMP ]] ; then
- SPELL_SRC_URL=$(quill_fmxml_hunt_src_url $SPELL_NAME $TMP)
- break
- fi
- done
- SPELL_URL=$(curl -I $(quill_fmxml_grab_from_xml url_homepage $1) 2>&1|\
- grep Location|awk '{print $2}'|sed 's/\r//g')
-
- SPELL_LICENSE=$(quill_fmxml_grab_from_xml license $1)
- if grep -q "(.*)" <<< "$SPELL_LICENSE"; then
- SPELL_LICENSE=$(awk '{print $NF}' <<< "$SPELL_LICENSE" | tr -d '()')
+ SPELL_NAME=$(quill_fmxml_grab_from_xml projectname_short ${FRESHMEAT_FILE}
|tr 'A-Z' 'a-z')
+ SPELL_NAME="${SPELL_NAME:-$1}"
+ SPELL_DESC_NAME="${SPELL_NAME}"
+ local tmp_url
+ for each in url_bz2 url_tgz url_zip; do
+ tmp_url="$(quill_fmxml_grab_from_xml $each ${FRESHMEAT_FILE})"
+ if [[ $tmp_url ]] ; then
+ SPELL_SRC_URL="$(grep "$each" ${FRESHMEAT_PROJECT_FILE} | grep -v
"\<img\ src\>" | sed -e "s: *<a href.*\">\(.*\)</a><br>:\1:")"
+ break
fi
- SPELL_SHORT_DESCRIPTION=$(quill_fmxml_grab_from_xml desc_short $1|sed
's/\r//g')
- quill_fmxml_grab_from_xml desc_full $1 |fmt >
${QUILL_TMP_DIR}/${SPELL_NAME}
+ done
+ SPELL_URL="$(grep "url_homepage" ${FRESHMEAT_PROJECT_FILE} | grep -v
"\<img\ src\>" | sed -e "s: *<a href.*\">\(.*\)</a><br>:\1:")"
+ SPELL_LICENSE="$(quill_fmxml_grab_from_xml license ${FRESHMEAT_FILE})"
+ if grep -q "(.*)" <<< "${SPELL_LICENSE}"; then
+ SPELL_LICENSE="$(awk '{print $NF}' <<< "$SPELL_LICENSE" | tr -d '()')"
fi
+ SPELL_SHORT_DESCRIPTION="$(quill_fmxml_grab_from_xml desc_short
${FRESHMEAT_FILE}|sed 's/\r//g')"
+ quill_fmxml_grab_from_xml desc_full ${FRESHMEAT_FILE} |fmt >
${QUILL_TMP_DIR}/${SPELL_NAME}
}
#---
##
diff --git a/var/lib/quill/modules/libperlcpan
b/var/lib/quill/modules/libperlcpan
index 2f3e7e3..0d08513 100644
--- a/var/lib/quill/modules/libperlcpan
+++ b/var/lib/quill/modules/libperlcpan
@@ -8,8 +8,16 @@ function quill_get_cpan_meta()
[ ! -f $QUILL_TMP_DIR/03modlist.data.gz ]
then
cd $QUILL_TMP_DIR
- wget -O 02packages.details.txt.gz
ftp://ftp.perl.org/pub/CPAN/modules/02packages.details.txt.gz
- wget -O 03modlist.data.gz
ftp://ftp.perl.org/pub/CPAN/modules/03modlist.data.gz
+ message "Attempting to retrive 02packages.details.txt.gz..."
+ wget -q \
+ -O 02packages.details.txt.gz \
+ ftp://ftp.perl.org/pub/CPAN/modules/02packages.details.txt.gz ||
+ (error_msg "Error: unable to fetch 02packages.details.txt.gz" &&
return 1)
+ message "Attempting to retrive 03modlist.data.gz..."
+ wget -q \
+ -O 03modlist.data.gz \
+ ftp://ftp.perl.org/pub/CPAN/modules/03modlist.data.gz ||
+ (error_msg "Error: unable to fetch 03modlist.data.gz" && return 1)
fi
eval QUILL_CPAN_INFO="( $(zgrep -w -E "^${1}"
$QUILL_TMP_DIR/02packages.details.txt.gz) )"
eval QUILL_CPAN_METADATA="( $(zgrep -w -B1 -A9 -E "^'${1}'"
$QUILL_TMP_DIR/03modlist.data.gz | tr -d "[]" | sed 's:,$::g' ) )"
diff --git a/var/lib/quill/modules/librubyraa
b/var/lib/quill/modules/librubyraa
index 3f65e08..89ef8ed 100644
--- a/var/lib/quill/modules/librubyraa
+++ b/var/lib/quill/modules/librubyraa
@@ -20,7 +20,8 @@ function quill_raa_core(){
message "Attempting to retrive project page..."
wget -q -O $raa_project_file.tmp \
http://raa.ruby-lang.org/project/$project/ || \
- error_msg "Error: unable to fetch project page"
+ error_msg "Error: unable to fetch project page" && \
+ return 1
sed -e "s:<a href=\"\(.*\)\".*:\1:g" \
-e ":top;/<.*>/{;s/<[^<>]*>//g;t top;};/</{;N;b top;}" \
$raa_project_file.tmp | \



  • [SM-Commit] GIT changes to master quill by Andra?? Levstik (0aa2e6a72ff48b963bf6ef41c478e20966f56d18), Andra?? Levstik, 02/24/2007

Archive powered by MHonArc 2.6.24.

Top of Page