spider

#!/bin/php

<?php
require_once('./lib/languages.php');
require_once('./lib/function.php');

define('IMAGE_DIR', 'images');
define('API_DIR', 'api');
define('ORIGIN_DIR', 'origin'); 
define('VIRGULE', '/');
define('HYPHEN', '-');
define('EXT_IMAGE','.jpg');
define('EXT_JSON', '.json');
$opt = getopt('', array('debug', 'sms', 'idx:','n:'));
//var_dump($opt);
$is_debug = isset($opt['debug']);
$is_sms = isset($opt['sms']);
$idx = isset($opt['idx']) ? $opt['idx'] : 0;
$n = isset($opt['n']) ? $opt['n'] : 1;
if (!$is_debug) {
    //清除未提交数据，防止脏数据
    exec('git add --all .');
    exec('git reset --hard HEAD');
    //切换git用户信息
    exec('git config --local user.name "Spider"');
    exec('git config --local user.email "spider@klavor.com"');
    //更新git代码
    exec('git pull');
}
$global_domain = 'https://global.bing.com';
$bing_domain = 'https://www.bing.com';
$algo = 'sha256';
$languages = get_languages();
//var_dump($languages);
if ($is_debug) {
    $languages = array("zh-CN"=>"CN");
}
//采集结果统计
$hd_count = 0;
$tmb_count = 0;
$api_count = 0;
$failure_count = 0;
$total_count = 0;
$hd_hash_array = array();
$tmb_hash_array = array();
foreach ($languages as $lang_region => $region) {
//    echo $lang_region.':'.$region.'
//';
    $bing_api = $global_domain . '/HPImageArchive.aspx?setmkt='.$lang_region.'&setlang=' . $lang_region . '&ensearch=0&format=js&idx='.$idx.'&n='.$n.'&pid=hp&quiz=1$&og=1&uhd=0';
//    echo 'bing_api = '.$bing_api.'
//';
    $response = request($bing_api);
    if ($response === false) {
        //没有请求到结果则继续执行循环的下一次操作
        //TODO 日志记录
        $failure_count++;
        continue;
    }
    
    $bing_obj = json_decode($response, false);
    $images = $bing_obj->images;
    foreach ($images as $image) {
        $total_count++;
        
        //取出变量
        $title         = $image->title;
        $caption       = $image->caption;
        $desc          = $image->desc;
        $copyrightonly = $image->copyrightonly;
        $copyright     = $image->copyright;
        $copyrightlink = $image->copyrightlink;
        $url           = $image->url;
        $urlbase       = $image->urlbase;
        $startdate     = $image->startdate;
        $fullstartdate = $image->fullstartdate;
        $enddate       = $image->enddate;
        $quiz          = $image->quiz;
        $hsh           = $image->hsh;
        $image_og      = $image->og;
        $og_title      = $image_og->title;
        $og_desc       = $image_og->desc;
        $og_img        = $image_og->img;
        $og_hash       = $image_og->hash;
        
        //不知道什么原因，有些国家地区返回的值晚一天
        $year  = substr($startdate, 0, 4);
        $month = substr($startdate, 4, 2);
        $day   = substr($startdate, -2);
    
//        echo 'year = '.$year.'; month = '.$month.'; day = '.$day.'
//';
        
        //获取bing原文件名。：OHR.FeatherSerpent_ZH-CN5706017355
        $image_name = substr(strrchr($urlbase, '='), 1) . '.jpg';
        //拼接对外公布的接口数据
        $api_obj                = new stdClass();
        $api_obj->title         = $title;
        $api_obj->caption       = $caption;
        $api_obj->desc          = $desc;
        $api_obj->copyright     = $copyright;
        $api_obj->copyrightlink = $copyrightlink;
        $api_obj->date          = $startdate;
        $api_obj_image          = new stdClass();
        $api_obj_image->name    = $image_name;
        $api_obj_image->hd      = null;
        $api_obj_image->tmb     = null;
        $api_obj_image->binghd  = null;
        $api_obj_image->bingtmb = $og_img;
        $api_obj->image         = $api_obj_image;
        

        $hash_key = substr($image_name, 0, strrpos($image_name, '_'));

        //拼接图片地址
        $image_url = $bing_domain.$url;
        $api_obj_image->binghd = $image_url;
        
        if (!array_key_exists($hash_key, $hd_hash_array)) {
            //请求数据
            $image_response = request($image_url);
            if ($image_response !== false) {
                $image_hash = strtoupper(hash($algo, $image_response));
                $hd_hash_array[$hash_key] = $image_hash;
                $image_path = IMAGE_DIR.VIRGULE.$image_hash.EXT_IMAGE;
                $image_save_result = check_image_and_save($image_path, $image_response);
                if ($image_save_result !== false) {
                    $hd_count++;
                }
                $api_obj_image->hd = VIRGULE.$image_path;
            }
        } else {
            $hd_count++;
            $image_hash = $hd_hash_array[$hash_key];
            $api_obj_image->hd = VIRGULE.IMAGE_DIR.VIRGULE.$image_hash.EXT_IMAGE;
        }


        if (!array_key_exists($hash_key, $tmb_hash_array)) {
            //TMB图下载
            $og_image_response =request($og_img);
            if ($image_response !== false) {
                $og_image_hash = strtoupper(hash($algo, $og_image_response));
                $tmb_hash_array[$hash_key] = $og_image_hash;
                $og_image_path = IMAGE_DIR.VIRGULE.$og_image_hash.EXT_IMAGE;
                $og_image_save_result = check_image_and_save($og_image_path, $og_image_response);
                if ($og_image_save_result !== false) {
                    $tmb_count++;
                }
                $api_obj_image->tmb = VIRGULE.$og_image_path;
            }
        } else {
            $tmb_count++;
            $og_image_hash = $tmb_hash_array[$hash_key];
            $api_obj_image->tmb = VIRGULE.IMAGE_DIR.VIRGULE.$og_image_hash.EXT_IMAGE;
        }

        //保存api数据
        //1.构建api存储路径
        $api_path = API_DIR
            .VIRGULE.$lang_region
            .VIRGULE.$year
            .VIRGULE.$month
            .VIRGULE.$year.HYPHEN.$month.HYPHEN.$day.EXT_JSON;
        
        //2.转换对象成Json
        $api_json = json_encode($api_obj, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
        //3.存储Json数据到本地
        $api_save_result = check_path_and_save($api_path, $api_json);
        if ($api_save_result !== false) {
            $api_count++;
        }
        echo 'Total: '.(sizeof($languages) * $n).'    Current: '.$total_count.'
';
    }
    
    //存储Bing原始数据
    $temp_image = $images[0];
    $temp_startdate = $temp_image->startdate;
    $year  = substr($temp_startdate, 0, 4);
    $month = substr($temp_startdate, 4, 2);
    $day   = substr($temp_startdate, -2);
    $origin_path = ORIGIN_DIR
        .VIRGULE.$lang_region
        .VIRGULE.$year
        .VIRGULE.$month
        .VIRGULE.$year.HYPHEN.$month.HYPHEN.$day.EXT_JSON;
    check_path_and_save($origin_path, $response);
}
if (!$is_debug) {
    //提交爬取数据
    exec('git add --all .');
    $comment = date('Y-m-d H:i');
    $comment .= ($hd_count == $total_count && $tmb_count == $total_count && $api_count == $total_count) ? '符合预期结果!
' : '数据有出入，需要进行数据检查!
';
    $comment .= '入参：
';
    $comment .= '    idx = '.$idx.'
';
    $comment .= '    n = '.$n.'
';
    $comment .= '结果：
';
    $comment .= '    HD\TMB\API理论值：'.$total_count.'
';
    $comment .= '    HD实际值        ：'.$hd_count.'
';
    $comment .= '    TMB实际值       ：'.$tmb_count.'
';
    $comment .= '    API实际值       ：'.$api_count.'
';
    $comment .= '    接口失败值      ：'.$failure_count.'
';
    exec('git commit -m "'.$comment.'"');
    exec('git push');
    //切换git用户信息
    exec('git config --local user.name "Klavor Lee"');
    exec('git config --local user.email "lee@klavor.com"');
}
if ($is_sms) {
    exec('../sendSMS');
}