PHP5 recursive URL-crawler


/ Published in: PHP
Save to your folder(s)



Copy this code and paste it in your HTML
  1. <?php
  2. class crawler{
  3. private $_depth=5;
  4. private $_urls=array();
  5.  
  6.  
  7. function extract_links($url)
  8. {
  9. if(!$this->_started){
  10. $this->_started=1;
  11. $curr_depth=0;
  12. }else{
  13. $curr_depth++;
  14. }
  15. if($curr_depth<$this->_depth)
  16. {
  17. $data=file_get_contents($url);
  18. if(preg_match_all('/((?:http|https):\/\/(?:www\.)*(?:[a-zA-Z0-9_\-]{1,15}\.+[a-zA-Z0-9_]{1,}){1,}(?:[a-zA-Z0-9_\/\.\-\?\&\:\%\,\!\;]*))/',$data,$urls12))
  19. {
  20. foreach($urls12[0] as $k=>$v){
  21. $check=get_headers($v,1);
  22. if(strstr($v,$url) && $check[0]=='HTTP/1.1 200 OK' && !array_search($v,$this->_urls) && $curr_depth<$this->_depth){
  23. $this->_urls[]=$v;
  24. $this->extract_links($v);
  25. }
  26. }
  27. }
  28. }
  29. return $this->_urls;
  30. }
  31. }
  32. ?>

URL: http://e-code.tnt43.com

Report this snippet


Comments

RSS Icon Subscribe to comments

You need to login to post a comment.