curlServiceFactory = $curlServiceFactory;
$this->config = $config;
}
/**
* @param string $httpHeader
* @param string $url
* @return array
*/
public function parseCookie($httpHeader, $url)
{
//convert cookie string to array
$parts = explode(";", $httpHeader);
$cookie = [];
foreach ($parts as $v) {
$v = trim($v);
parse_str($v, $vv);
$cookie[key($vv)] = reset($vv);
}
reset($cookie);
$cookie['key'] = key($cookie);
$cookie['value'] = array_shift($cookie);
;
if (!isset($cookie['path'])) {
$cookie['path'] = '/';
}
if (!isset($cookie['domain'])) {
$cookie['domain'] = parse_url($url, PHP_URL_HOST);
}
return $cookie;
}
/**
* @param string $url
* @param string $content
*/
public function parseCookies($url, $content)
{
preg_match_all('/^Set-Cookie:(.*);/mi', $content, $matches);
foreach ($matches[1] as $httpHeader) {
$cookie = $this->parseCookie($httpHeader, $url);
$path = $cookie['domain'].$cookie['path'];
if (!isset($this->cookies[$path])) {
$this->cookies[$path] = [];
}
if ($cookie['value'] == "deleted") {
unset($this->cookies[$path][$cookie['key']]);
} else {
$this->cookies[$path][$cookie['key']] = $cookie['value'];
}
}
}
/**
* @param string $url
* @param string $sessionDataCookie
* @param OutputInterface $output
* @param string|bool $userAgent
* @param string|bool $storeCode
*
* @return array
*/
public function getUrls($url, $sessionDataCookie, OutputInterface $output, $customerData = [], $userAgent = false, $storeCode = false)
{
$result = $this->makeRequest($url, $sessionDataCookie, $customerData, $userAgent, $storeCode);
$output->writeln("".$result['curl']."");
$response = $result['response'];
$code = $response->getCode();
$style = '';
switch (substr((string)$code, 0, 1)) {
case 1:
$style = 'bg=magenta';
break;
case 2:
$style = 'bg=green';
break;
case 3:
$style = 'bg=yellow';
break;
case 4:
case 5:
$style = 'bg=red';
break;
default:
$style = 'bg=default';
}
$output->writeln( 'Response code: <' . $style . '> ' . $code . ' ');
if ($code >= 400) {
$output->writeln("Can't open URL.");
if ($code == 401) {
$output->writeln("Authorization Required. Set HTTP Access info in the extension settings");
}
return [];
}
if ($this->config->isIgnoredUri($url)) {
$output->writeln('Info: Ignored by "Ignored Pages" exceptions.');
}
if (
($userAgent && $this->config->isIgnoredUserAgent($userAgent))
|| (!$userAgent && $this->config->isIgnoredUserAgent(UserAgentInterface::DESKTOP_USER_AGENT))
) {
$output->writeln('Info: Ignored by "Ignored User-Agents" exceptions.');
}
$content = $response->getBody();
$this->parseCookies($url, $content);
$dom = new \DOMDocument();
libxml_use_internal_errors(true);
if ($content) {
$dom->loadHTML($content);
}
libxml_clear_errors();
$links = $dom->getElementsByTagName('a');
$urlInfo = parse_url($url);
$baseUrl = $urlInfo['scheme'] . '://' . $urlInfo['host'];
$result = [];
/** @var \DOMElement $link */
foreach ($links as $link) {
$l = $link->getAttribute('href');
;
if (strpos($l, "/") === 0) { //absolute url like "/xxx/yyy"
$l = $baseUrl.$l;
}
$result[] = $l;
}
return $result;
}
/**
* @param string $url
* @param string $sessionDataCookie
* @param string|bool $userAgent
* @param string|bool $storeCode
*
* @return array
*/
public function makeRequest($url, $sessionDataCookie, $customerData = [], $userAgent = false, $storeCode = false)
{
$curlService = $this->curlServiceFactory->create();
$channel = $curlService->initChannel();
$channel->setUrl($url);
$channel->setOption(CURLOPT_FOLLOWLOCATION, true);
$channel->setOption(CURLOPT_HEADER, true);
$userAgent = $userAgent ?: UserAgentInterface::DESKTOP_USER_AGENT;
$channel->setUserAgent($userAgent);
if ($storeCode) {
$channel->addCookie('store', $storeCode);
}
// apply cookies
// we sort array to make sure that
// cookies with path x.com/ are applied before cookies with path x.com/de/
$keys = array_map('strlen', array_keys($this->cookies));
array_multisort($keys, SORT_DESC, $this->cookies);
foreach ($this->cookies as $domainPath => $v) {
// if cookie domain is not .x.com
if ($domainPath[0] !== ".") {
$domainPath = "://".$domainPath;
}
if (strpos($url, $domainPath) === false) {
continue;
}
foreach ($v as $k => $v2) {
$channel->addCookie($k, $v2);
}
}
if ($sessionDataCookie) {
$channel->addCookie(SessionServiceInterface::SESSION_COOKIE, $sessionDataCookie);
}
if ($this->config->isUsePhpSessionCookie()) {
$varyDataHash = sha1(SerializeService::encode($customerData));
$channel->addCookie(
'PHPSESSID',
substr(SessionServiceInterface::PHPSESSID_PREFIX . $varyDataHash, 0, 26)
);
}
$delay = $this->config->getDelay();
if ($delay > 0) {
usleep($delay * 1000);
}
$response = $curlService->request($channel);
return ['response' => $response, 'curl' => $channel->getCUrl()];
}
}