Initial commit

Functional, without SSO
This commit is contained in:
Jimmy Monin
2016-09-18 11:03:26 +02:00
commit 57708e3169
253 changed files with 30787 additions and 0 deletions

View File

@ -0,0 +1,287 @@
<?php
namespace AdService;
class Ad
{
protected $_id;
protected $_link;
protected $_link_mobile;
protected $_title;
protected $_description;
protected $_price;
protected $_currency = "";
protected $_date;
protected $_category;
protected $_country;
protected $_city;
protected $_professional;
protected $_thumbnail_link;
protected $_urgent;
/**
* @param int $id
* @return \AdService\Ad
*/
public function setId($id)
{
$this->_id = $id;
return $this;
}
/**
* @return int
*/
public function getId()
{
return $this->_id;
}
/**
* @param string $link
* @return \AdService\Ad
*/
public function setLink($link)
{
$this->_link = $link;
return $this;
}
/**
* @return string
*/
public function getLink()
{
return $this->_link;
}
/**
* @param string $link
* @return \AdService\Ad
*/
public function setLinkMobile($link)
{
$this->_link_mobile = $link;
return $this;
}
/**
* @return string
*/
public function getLinkMobile()
{
return $this->_link_mobile;
}
/**
* @param string $title
* @return \AdService\Ad
*/
public function setTitle($title)
{
$this->_title = $title;
return $this;
}
/**
* @return string
*/
public function getTitle()
{
return $this->_title;
}
/**
* @param string $description
* @return \AdService\Ad
*/
public function setDescription($description)
{
$this->_description = $description;
return $this;
}
/**
* @return string
*/
public function getDescription()
{
return $this->_description;
}
/**
* @param int $price
* @return \AdService\Ad
*/
public function setPrice($price)
{
// $this->_price = (int) preg_replace('/[^0-9]*/', '', $price);
$this->_price = $price;
return $this;
}
/**
* @return int
*/
public function getPrice()
{
return $this->_price;
}
/**
* @param string $currency
* @return Ad
*/
public function setCurrency($currency)
{
$this->_currency = $currency;
return $this;
}
/**
* @return string
*/
public function getCurrency()
{
return $this->_currency;
}
/**
* @param Zend_Date $date
* @return \AdService\Ad
*/
public function setDate($date)
{
$this->_date = $date;
return $this;
}
/**
* @return string
*/
public function getDate()
{
return $this->_date;
}
/**
* @param string $category
* @return \AdService\Ad
*/
public function setCategory($category)
{
$this->_category = $category;
return $this;
}
/**
* @return string
*/
public function getCategory()
{
return $this->_category;
}
/**
* @param string $county
* @return \AdService\Ad
*/
public function setCountry($county)
{
$this->_country = $county;
return $this;
}
/**
* @return string
*/
public function getCountry()
{
return $this->_country;
}
/**
* @param string $city
* @return \AdService\Ad
*/
public function setCity($city)
{
$this->_city = $city;
return $this;
}
/**
* @return string
*/
public function getCity()
{
return $this->_city;
}
/**
* @param bool $professional
* @return \AdService\Ad
*/
public function setProfessional($professional)
{
$this->_professional = $professional;
return $this;
}
/**
* @return bool
*/
public function getProfessional()
{
return $this->_professional;
}
/**
* @param string $thumbail
* @return \AdService\Ad
*/
public function setThumbnailLink($thumbail)
{
$this->_thumbnail_link = $thumbail;
return $this;
}
/**
* @return string
*/
public function getThumbnailLink()
{
return $this->_thumbnail_link;
}
/**
* @param bool $urgent
* @return \AdService\Ad
*/
public function setUrgent($urgent)
{
$this->_urgent = (bool)$urgent;
return $this;
}
/**
* @return bool
*/
public function getUrgent()
{
return $this->_urgent;
}
}

View File

@ -0,0 +1,8 @@
<?php
namespace AdService;
class Exception extends \Exception
{
}

View File

@ -0,0 +1,188 @@
<?php
namespace AdService;
class Filter
{
protected $min_id = 0;
protected $exclude_ids = array();
protected $price_min = -1;
protected $price_max = -1;
protected $price_strict = false;
protected $cities = "";
protected $categories = array();
public function __construct(array $options = array())
{
$this->setFromArray($options);
}
public function isValid(Ad $ad)
{
if (!$ad->getPrice() && $this->price_strict) {
return false;
}
if ($ad->getPrice()) {
if ($this->price_min != -1 && $ad->getPrice() < $this->price_min
|| $this->price_max != -1 && $ad->getPrice() > $this->price_max) {
return false;
}
}
$city = mb_strtolower($ad->getCity());
$country = mb_strtolower($ad->getCountry());
if ($this->cities && !in_array($city, $this->cities) && !in_array($country, $this->cities)) {
return false;
}
if ($this->categories && !in_array($ad->getCategory(), $this->categories)) {
return false;
}
return true;
}
public function setFromArray(array $options)
{
foreach ($options AS $option => $value) {
$method = "set".str_replace(" ", "", ucwords(
str_replace("_", " ", $option)));
if (method_exists($this, $method)) {
$this->$method($value);
}
}
}
/**
* @param int $min_id
* @return Filter
*/
public function setMinId($min_id)
{
$this->min_id = $min_id;
return $this;
}
/**
* @return int
*/
public function getMinId()
{
return $this->min_id;
}
/**
* @param array $exclude_ids
* @return Filter
*/
public function setExcludeIds($exclude_ids)
{
$this->exclude_ids = $exclude_ids;
return $this;
}
/**
* @return array
*/
public function getExcludeIds()
{
return $this->exclude_ids;
}
/**
* @param int $price_min
* @return Filter
*/
public function setPriceMin($price_min)
{
$this->price_min = $price_min;
return $this;
}
/**
* @return int
*/
public function getPriceMin()
{
return $this->price_min;
}
/**
* @param int $price_max
* @return Filter
*/
public function setPriceMax($price_max)
{
$this->price_max = $price_max;
return $this;
}
/**
* @return int
*/
public function getPriceMax()
{
return $this->price_max;
}
/**
* @param bool $price_strict
* @return Filter
*/
public function setPriceStrict($price_strict)
{
$this->price_strict = $price_strict;
return $this;
}
/**
* @return bool
*/
public function getPriceStrict()
{
return $this->price_strict;
}
/**
* @param array $cities
* @return Filter
*/
public function setCities(array $cities)
{
$this->cities = $cities;
return $this;
}
/**
* @return array
*/
public function getCities()
{
return $this->cities;
}
/**
* @param array $categories
* @return Filter
*/
public function setCategories(array $categories)
{
$this->categories = $categories;
return $this;
}
/**
* @return array
*/
public function getCategories()
{
return $this->categories;
}
}

View File

@ -0,0 +1,19 @@
<?php
namespace AdService\Parser;
use AdService\Filter;
abstract class AbstractParser extends \DOMDocument
{
public function __construct()
{
libxml_use_internal_errors(true);
}
/**
* @param string $content
* @param Filter $filter
*/
abstract public function process($content, Filter $filter = null);
}

View File

@ -0,0 +1,177 @@
<?php
namespace AdService\Parser;
use AdService\Filter;
use AdService\Ad;
class Lbc extends AbstractParser
{
protected static $months = array(
"jan" => 1, "fév" => 2, "mars" => 3, "avr" => 4,
"mai" => 5, "juin" => 6, "juillet" => 7, "août" => 8,
"sept" => 9, "oct" => 10, "nov" => 11,
"déc" => 12
);
protected $scheme;
public function process($content, Filter $filter = null, $scheme = "http") {
if (!$content) {
return;
}
$this->scheme = $scheme;
$this->loadHTML($content);
$timeToday = strtotime(date("Y-m-d")." 23:59:59");
$dateYesterday = $timeToday - 24*3600;
$ads = array();
if ($filter) {
$exclude_ids = $filter->getExcludeIds();
/**
* Afin de garder une rétrocompatibilité, on prend en compte
* que $exclude_ids peut être numérique.
*/
if (!is_numeric($exclude_ids) && !is_array($exclude_ids)) {
unset($exclude_ids);
}
}
$adNodes = $this->getElementsByTagName("a");
foreach ($adNodes AS $result) {
// est-ce bien une annonce ?
if (false === strpos($result->getAttribute("class"), "list_item")) {
continue;
}
$ad = new Ad();
$ad->setProfessional(false)->setUrgent(false);
// pas d'ID, pas d'annonce
if (!preg_match('/([0-9]+)\.htm.*/', $result->getAttribute("href"), $m)) {
continue;
}
// permet d'éliminer les annonces déjà envoyées.
if (isset($exclude_ids)) {
if (is_numeric($exclude_ids)) {
/**
* Si $exclude_ids est numérique, alors détection
* à l'ancienne. Quand on rencontre l'ID de la
* dernière annonce, on stoppe la boucle.
*/
if ($m[1] == $exclude_ids) {
break;
}
} elseif (in_array($m[1], $exclude_ids)) {
continue;
}
}
// permet d'éliminer les annonces déjà envoyées.
if ($filter && $m[1] <= $filter->getMinId()) {
continue;
}
$ad->setLink($this->formatLink($result->getAttribute("href")))
->setId($m[1])
->setTitle($result->getAttribute("title"))
->setLinkMobile(str_replace(
array("http://www.", "https://www."),
array("http://mobile.", "https://mobile."),
$ad->getLink()
));
// recherche de l'image
foreach ($result->getElementsByTagName("span") AS $node) {
if ($src = $node->getAttribute("data-imgsrc")) {
$ad->setThumbnailLink($this->formatLink($src));
}
}
$i = 0;
foreach ($result->getElementsByTagName("p") AS $node) {
$class = (string) $node->getAttribute("class");
if (false !== strpos($class, "item_supp")) {
$value = trim($node->nodeValue);
if ($i == 0) { // catégorie
if (false !== strpos($value, "(pro)")) {
$ad->setProfessional(true);
}
$ad->setCategory(trim(str_replace("(pro)", "", $value)));
} elseif ($i == 1) { // localisation
if (false !== strpos($value, "/")) {
$value = explode("/", $value);
$ad->setCountry(trim($value[1]))
->setCity(trim($value[0]));
} else {
$ad->setCountry(trim($value));
}
} elseif ($i == 2) { // date de l'annonce + urgent
$spans = $node->getElementsByTagName("span");
if ($spans->length > 0) {
$ad->setUrgent(true);
$node->removeChild($spans->item(0));
$value = trim($node->nodeValue);
}
$dateStr = preg_replace("#\s+#", " ", $value);
$aDate = explode(' ', $dateStr);
$aDate[1] = trim($aDate[1], ",");
if (false !== strpos($dateStr, 'Aujourd')) {
$time = strtotime(date("Y-m-d")." 00:00:00");
} elseif (false !== strpos($dateStr, 'Hier')) {
$time = strtotime(date("Y-m-d")." 00:00:00");
$time = strtotime("-1 day", $time);
} else {
if (!isset(self::$months[$aDate[1]])) {
continue;
}
$time = strtotime(date("Y")."-".self::$months[$aDate[1]]."-".$aDate[0]);
}
$aTime = explode(":", $aDate[count($aDate) - 1]);
$time += (int)$aTime[0] * 3600 + (int)$aTime[1] * 60;
if ($timeToday < $time) {
$time = strtotime("-1 year", $time);
}
$ad->setDate($time);
}
$i++;
}
}
// recherche du prix
foreach ($result->getElementsByTagName("h3") AS $node) {
$class = (string) $node->getAttribute("class");
if (false !== strpos($class, "item_price")) {
if (preg_match("#[0-9 ]+#", $node->nodeValue, $m)) {
$ad->setPrice((int)str_replace(" ", "", trim($m[0])));
}
}
}
// exclure les annonces ne correspondant pas au filtre.
if ($filter && !$filter->isValid($ad)) {
continue;
}
$ads[$ad->getId()] = $ad;
}
return $ads;
}
protected function formatLink($link)
{
if (0 === strpos($link, "//")) {
$link = $this->scheme.":".$link;
}
return $link;
}
}

View File

@ -0,0 +1,137 @@
<?php
namespace AdService\Parser;
use AdService\Filter;
use AdService\Ad;
class Olx extends AbstractParser
{
protected static $months = array(
"jan" => 1, "fév" => 2, "mars" => 3, "апр." => 4,
"mai" => 5, "juin" => 6, "juillet" => 7, "août" => 8,
"sept" => 9, "oct" => 10, "nov" => 11,
"déc" => 12
);
public function process($content, Filter $filter = null) {
if (!$content) {
return;
}
$content = str_replace("<br/>", " ", $content);
$this->loadHTML($content);
$timeToday = strtotime(date("Y-m-d")." 23:59:59");
$dateYesterday = $timeToday - 24*3600;
$ads = array();
$tables = $this->getElementsByTagName("table");
$tableOffers = null;
foreach ($tables AS $table) {
if (false !== strpos($table->getAttribute("id"), "offers_table")) {
$tableOffers = $table;
break;
}
}
if (!$tableOffers) {
return array();
}
$adNodes = $tableOffers->getElementsByTagName("td");
foreach ($adNodes AS $adNode) {
if (false === strpos($adNode->getAttribute("class"), "offer")) {
continue;
}
$ad = new Ad();
$ad->setUrgent(false);
// aucun indicateur pour savoir si c'est un pro ou non.
$ad->setProfessional(false);
// permet d'éliminer les annonces déjà envoyées.
// @todo pour le moment, pas possible. Les IDs ne semblent pas
// numérique et incrémentals.
// if ($filter && $m[1] <= $filter->getMinId()) {
// continue;
// }
$rows = $adNode->getElementsByTagName("tr");
if (0 == $rows->length) {
continue;
}
$columns = $adNode->getElementsByTagName("td");
$row2_p = $rows->item(1)->getElementsByTagName("p");
// analyse de la date
$dateStr = preg_replace("#\s+#", " ", trim($row2_p->item(1)->nodeValue));
if (!$dateStr) {
continue;
}
$aDate = explode(' ', $dateStr);
if (false !== strpos($dateStr, 'Сегодня')) { // aujourd'hui
$time = strtotime(date("Y-m-d")." 00:00:00");
} elseif (false !== strpos($dateStr, 'Вчера')) {
$time = strtotime(date("Y-m-d")." 00:00:00");
$time = strtotime("-1 day", $time);
} else {
if (!isset(self::$months[$aDate[1]])) {
continue;
}
$time = strtotime(date("Y")."-".self::$months[$aDate[1]]."-".$aDate[0]);
}
$timeStr = $aDate[count($aDate) - 1];
if (false !== $pos = mb_strpos($dateStr, ":")) {
$time += (int)mb_substr($dateStr, $pos - 2, 2) * 3600;
$time += (int)mb_substr($dateStr, $pos + 1, 2) * 60;
if ($timeToday < $time) {
$time = strtotime("-1 year", $time);
}
}
$ad->setDate($time);
// image
$img = $columns->item(0)->getElementsByTagName("img");
if ($img->length) {
$ad->setThumbnailLink(str_replace("94x72", "644x461", $img->item(0)->getAttribute("src")));
}
// titre + lien
$link = $adNode->getElementsByTagName("h3")->item(0)->getElementsByTagName("a")->item(0);
if ($link) {
$ad->setTitle(trim($link->nodeValue));
$ad->setLink($link->getAttribute("href"));
}
// urgent
if (false !== strpos($adNode->nodeValue, "Срочно")) {
$ad->setUrgent(true);
}
// lieu
$ad->setCity(trim($row2_p->item(0)->nodeValue));
// catégorie
$ad->setCategory(trim($columns->item(1)->getElementsByTagName("p")->item(0)->nodeValue));
if (!preg_match("#ID([^.]+)\.html#", $ad->getLink(), $m)) {
continue;
}
$ad->setId(base_convert($m[1], 32, 10));
$priceColumn = trim($columns->item(2)->nodeValue);
if (preg_match('#(?<price>[0-9\s]+)\s+(?<currency>грн|\$|€)#imsU', $priceColumn, $m)) {
$ad->setPrice((int) str_replace(" ", "", $m["price"]))
->setCurrency($m["currency"]);
}
if ($filter && !$filter->isValid($ad)) {
continue;
}
$ads[$ad->getId()] = $ad;
}
return $ads;
}
}

View File

@ -0,0 +1,125 @@
<?php
namespace AdService\Parser;
use AdService\Filter;
use AdService\Ad;
class Seloger extends AbstractParser
{
public function process($content, Filter $filter = null) {
if (!$content) {
return;
}
// pourquoi est-ce nécessaire ?! Je n'ai pas encore trouvé la raison.
$content = utf8_decode($content);
$this->loadHTML($content);
$timeToday = strtotime(date("Y-m-d")." 23:59:59");
$dateYesterday = $timeToday - 24*3600;
$ads = array();
$sections = $this->getElementsByTagName("section");
$section_results = null;
foreach ($sections AS $section) {
if (false !== strpos($section->getAttribute("class"), "liste_resultat")) {
$section_results = $section;
break;
}
}
if (!$section_results) {
return array();
}
if ($filter) {
$exclude_ids = $filter->getExcludeIds();
/**
* Afin de garder une rétrocompatibilité, on prend en compte
* que $exclude_ids peut être numérique.
*/
if (!is_numeric($exclude_ids) && !is_array($exclude_ids)) {
unset($exclude_ids);
}
}
$adNodes = $section_results->getElementsByTagName("article");
foreach ($adNodes AS $adNode) {
if (!$id = (int) $adNode->getAttribute("data-listing-id")) {
continue;
}
// permet d'éliminer les annonces déjà envoyées.
if (isset($exclude_ids)) {
if (is_numeric($exclude_ids)) {
/**
* Si $exclude_ids est numérique, alors détection
* à l'ancienne. Quand on rencontre l'ID de la
* dernière annonce, on stoppe la boucle.
*/
if ($id == $exclude_ids) {
break;
}
} elseif (in_array($id, $exclude_ids)) {
continue;
}
}
$ad = new Ad();
$ad->setUrgent(false)
->setId($id);
// aucun indicateur pour savoir si c'est un pro ou non.
$ad->setProfessional(false);
// image
$imgs = $adNode->getElementsByTagName("img");
if ($imgs->length) {
foreach ($imgs AS $img) {
if (false !== strpos($img->getAttribute("class"), "listing_photo")) {
$ad->setThumbnailLink(
str_replace(
array("c175", "c250"),
"b600",
$img->getAttribute("src"))
);
break;
}
}
}
// titre + lien + lieu
$link = $adNode->getElementsByTagName("h2")->item(0)
->getElementsByTagName("a")->item(0);
if ($link) {
$city = $link->getElementsByTagName("span")->item(0);
if ($city) {
// lieu
$ad->setCity(trim($city->nodeValue));
}
$ad->setTitle(trim($link->nodeValue));
$ad->setLink($link->getAttribute("href"));
}
$links = $adNode->getElementsByTagName("a");
if ($links->length) {
foreach ($links AS $link) {
$classCSS = $link->getAttribute("class");
if (false !== strpos($classCSS, "amount")) {
$ad->setPrice((int) preg_replace("#[^0-9]*#", "", $link->nodeValue));
}
}
}
if ($filter && !$filter->isValid($ad)) {
continue;
}
$ads[$ad->getId()] = $ad;
}
return $ads;
}
}

View File

@ -0,0 +1,24 @@
<?php
namespace AdService;
class ParserFactory
{
/**
* @param string $url
* @return \AdService\Parser\AbstractParser
*/
public static function factory($url)
{
if (false !== strpos($url, "leboncoin.fr")) {
return new Parser\Lbc();
}
if (false !== strpos($url, "olx.ua")) {
return new Parser\Olx();
}
if (false !== strpos($url, "seloger.com")) {
return new Parser\Seloger();
}
throw new Exception("No parser found");
}
}

View File

@ -0,0 +1,46 @@
<?php
namespace AdService\SiteConfig;
abstract class AbstractSiteConfig
{
/**
* Nom du site
* @var string
*/
protected $site_name = "";
/**
* URL du site
* @var string
*/
protected $site_url = "";
/**
* Les devises acceptées par le site.
* @var array
*/
protected $currencies = array("");
/**
* Indique si l'information d'une annonce pro ou particulier est disponible
* dans la liste d'annonce.
* @var bool
*/
protected $pro_visible = true;
/**
* Indique si le site fourni une date par annonce.
* @var bool
*/
protected $has_date = true;
/**
* @param string $name
* @return mixed
*/
public function getOption($name)
{
return isset($this->$name) ? $this->$name : null;
}
}

View File

@ -0,0 +1,9 @@
<?php
namespace AdService\SiteConfig;
class Lbc extends AbstractSiteConfig
{
protected $site_name = "LeBonCoin";
protected $site_url = "https://www.leboncoin.fr";
}

View File

@ -0,0 +1,11 @@
<?php
namespace AdService\SiteConfig;
class Olx extends AbstractSiteConfig
{
protected $site_name = "Olx";
protected $site_url = "http://olx.ua";
protected $currencies = array("", "\$", "грн");
protected $pro_visible = false;
}

View File

@ -0,0 +1,11 @@
<?php
namespace AdService\SiteConfig;
class Seloger extends AbstractSiteConfig
{
protected $site_name = "SeLoger";
protected $site_url = "http://www.seloger.com";
protected $pro_visible = false;
protected $has_date = false;
}

View File

@ -0,0 +1,33 @@
<?php
namespace AdService;
class SiteConfigFactory
{
protected static $instances;
/**
* @param string $url
* @param bool $singleton en général, c'est la même config pour tous.
* @return \AdService\SiteConfig\AbstractSiteConfig
*/
public static function factory($url, $singleton=true)
{
if (false !== strpos($url, "leboncoin.fr")) {
$className = 'AdService\SiteConfig\Lbc';
} elseif (false !== strpos($url, "olx.ua")) {
$className = 'AdService\SiteConfig\Olx';
} elseif (false !== strpos($url, "www.seloger.com")) {
$className = 'AdService\SiteConfig\Seloger';
} else {
throw new Exception("No config found");
}
if ($singleton) {
if (!isset(self::$instances[$className])) {
self::$instances[$className] = new $className;
}
return self::$instances[$className];
}
return new $className;
}
}