'app-plug-in
'menu News Reader

#######################################################################
# $Id: snarf.bas,v 1.4 2006/01/28 07:24:56 zeeb90au Exp $
# fetch web pages for off-line reading
#######################################################################

# strip the given tag from the html text
sub removeTag(byref s, tag) 
  local i, i_end, s_len, end_tag
  
  end_tag = "</"+tag+">"
  tag = "<"+tag
  s_len = len(s)
  
  for i = 1 to s_len
    i = instr(i, s, tag)
    if (i = 0) then
      exit for
    fi
    i_end = instr(i, s, end_tag)
    if (i_end = 0) then
      i_end = s_len
    fi
    i_end += len(end_tag)
    s_end = if(i_end<s_len, mid(s, i_end), "")
    s = left(s, i-1) + s_end
    s_len -= (i_end-i)
  next i
end  

# return an array of anchor links from the html text
func getLinks(byref s)
  local i, iend, s_len, links, end_ch, ahref
  
  s_len = len(s)
  i = 1
  
  for i = 1 to s_len
    i = instr(i, s, "<a href")
    if (i = 0) then
      exit for
    fi
    i += 8 'advance past tag
    end_ch = ">"
    if i = instr(i, s, chr(34)) then
      end_ch = chr(34) 'double quote
      i++
    elif i = instr(i, s, chr(39)) then
      end_ch = chr(39) 'single quote
      i++
    fi

    iend = instr(i, s, end_ch)
    if (iend = 0) then
      exit for
    fi
    ahref = mid(s, i, iend-i)
    if (instr(1, ahref, "#") != 1) then
      links << ahref
    fi
    
  next i
  getLinks=links
end func

# connect to the given url and retrive the page text
func snarf(url)
  local s

  open url as #2
  if (eof(2)) then
    ? "Connection failed: "+url
    exit
  fi

  ? "Snarfing " + url
  tload #2, s
  removeTag s, "script"
  removeTag s, "style"
  removeTag s, "noscript"
  close #2
  snarf = s
end

# create the local file system path
sub mkpath(s)
  local s_len, i, path

  s_len = len(s)
  i = 2 'skip initial /
  while (i < s_len)
    i = instr(i, s, "/")
    if (i = 0) then
      exit sub
    fi

    path = left(s, i-1)
    if exist(path) = 0 then
      mkdir path
    fi
    i++
  wend
end

# fetch pages from the given site
sub fetch
  
  mkpath indexPage

  'fetch the index page
  open indexPage for output as #1
  s = snarf(indexURL)
  links = getLinks(s)
  's = translate(s, "/handheld", "|")  
  print #1, s
  close #1
  
  'fetch the sub-pages
  for href in links
    path = cacheHome+href 
    mkpath path
    open path for output as #1
    s = snarf(homeURL+href)
'    s = translate(s, "/handheld", "|")
    print #1, s
    close #1
  next
  
  ? "Done"
end

# process button clicks - command =value of onclick argument
sub main
   cmd="smh"
   button 0,2,0,1,v,"Select site","label"
   button 0,2,1,2,cmd,"smh|age","choice"
   button 0,1,2,3,ok,"OK"
   button 1,2,2,3,can,"Cancel"
   doform  
   if can="Cancel" then
      exit sub
   fi
  select case cmd
  case "smh"
    cacheHome = ENV("HOME")+"/.smallbasic/cache/smh"
    homeURL = "http://www.smh.com.au"
    indexURL = homeURL+"/handheld/index.html"
    indexPage = cacheHome+"/handheld/index.html"
    fetch
  case "age"
    cacheHome = ENV("HOME")+"/.smallbasic/cache/age"
    homeURL = "http://www.theage.com.au"
    indexURL = homeURL+"/handheld/index.html"
    indexPage = cacheHome+"/handheld/index.html"
    fetch
  end select
end

main