Added scrape to api - Must be logged in to use - wont show for guests

api/functions/api-functions.php

@@ -1299,4 +1299,44 @@ function youtubeSearch($query)
 		$results = json_decode($response->body, true);
 	}
 	return ($results) ? $results : false;
+}
+
+function scrapePage($array)
+{
+	try {
+		$url = $array['data']['url'] ?? false;
+		$type = $array['data']['type'] ?? false;
+		if (!$url) return array(
+			'result' => 'Error',
+			'data' => 'No URL'
+		);
+		$url = qualifyURL($url);
+		$data = array(
+			'full_url' => $url,
+			'drill_url' => qualifyURL($url, true)
+		);
+		$options = array('verify' => false);
+		$response = Requests::get($url, array(), $options);
+		$data['response_code'] = $response->status_code;
+		if ($response->success) {
+			$data['result'] = 'Success';
+			switch ($type) {
+				case 'html':
+					$data['data'] = html_entity_decode($response->body);
+					break;
+				case 'json':
+					$data['data'] = json_decode($response->body);
+					break;
+				default:
+					$data['data'] = $response->body;
+			}
+			return $data;
+		}
+	} catch (Requests_Exception $e) {
+		return array(
+			'result' => 'Error',
+			'data' => $e->getMessage()
+		);
+	};
+	return array('result' => 'Error');
 }

api/functions/normal-functions.php

@@ -472,14 +472,17 @@ function qualifyURL($url, $return = false)
 	$port = (isset($digest['port']) ? ':' . $digest['port'] : '');
 	// Path
 	$path = (isset($digest['path']) ? $digest['path'] : '');
+	// Query
+	$query = (isset($digest['query']) ? '?' . $digest['query'] : '');
 	// Output
 	$array = array(
 		'scheme' => $scheme,
 		'host' => $host,
 		'port' => $port,
-		'path' => $path
+		'path' => $path,
+		'query' => $query
 	);
-	return ($return) ? $array : $scheme . '://' . $host . $port . $path;
+	return ($return) ? $array : $scheme . '://' . $host . $port . $path . $query;
 }
 
 function getServerPath($over = false)

api/index.php

@@ -35,9 +35,8 @@ if (!in_array($function, $approvedFunctionsBypass)) {
 }
 $result['request'] = key($_GET);
 $result['params'] = $_POST;
-
 //Custom Page Check
-if(strpos($function,'v1_custom_page_') !== false){
+if (strpos($function, 'v1_custom_page_') !== false) {
 	$endpoint = explode('v1_custom_page_', $function)[1];
 	$function = 'v1_custom_page';
 }
@@ -1293,11 +1292,11 @@ switch ($function) {
 				break;
 		}
 		$token = validateToken($token);
-		if($token){
+		if ($token) {
 			$result['status'] = 'success';
 			$result['statusText'] = 'success';
 			$result['data'] = $token;
-		}else{
+		} else {
 			$result['status'] = 'error';
 			$result['statusText'] = 'Token not validated or empty';
 		}
@@ -1388,7 +1387,7 @@ switch ($function) {
 	case 'v1_custom_page':
 		switch ($method) {
 			case 'GET':
-				$customPage = 'customPage'.ucwords($endpoint);
+				$customPage = 'customPage' . ucwords($endpoint);
 				$result['status'] = 'success';
 				$result['statusText'] = 'success';
 				$result['data'] = $$customPage;
@@ -1413,6 +1412,25 @@ switch ($function) {
 				break;
 		}
 		break;
+	case 'v1_scrape':
+		switch ($method) {
+			case 'POST':
+				if (qualifyRequest(998)) {
+					$result['status'] = 'success';
+					$result['statusText'] = 'success';
+					$result['data'] = scrapePage($_POST);
+				} else {
+					$result['status'] = 'error';
+					$result['statusText'] = 'API/Token invalid or not set';
+					$result['data'] = null;
+				}
+				break;
+			default:
+				$result['status'] = 'error';
+				$result['statusText'] = 'The function requested is not defined for method: ' . $method;
+				break;
+		}
+		break;
 	default:
 		//No Function Available
 		$result['status'] = 'error';
@@ -1427,9 +1445,9 @@ if (!$result) {
 $result['generationDate'] = $GLOBALS['currentTime'];
 $result['generationTime'] = formatSeconds(timeExecution());
 //Set HTTP Code
-if($result['statusText'] == "API/Token invalid or not set"){
+if ($result['statusText'] == "API/Token invalid or not set") {
 	http_response_code(401);
-}else{
+} else {
 	http_response_code(200);
 }
 //return JSON array

js/functions.js

@@ -2239,16 +2239,47 @@ function buildTwoFA(current){
         </div>
     </div>
     `;
-
-
-
-
-
-
-
-
     return element;
 }
+function scrapeCall(){
+    // Define the URL to scrape [only supports GET at the moment
+    var url = 'https://api.github.com/users/causefx/repos';
+    // Define callbacks variable first
+    var callbacks = $.Callbacks();
+    // Add functions that will deal with the data
+    callbacks.add( scrapeFunction );
+    // Call the API function to scrape the page you want [types = 'json' or 'html']
+    scrapeAPI(url, callbacks, 'json');
+}
+function scrapeFunction(data){
+    // Here you would do whatever you like
+    if(data.data.result == 'Success'){
+        console.log('Success!!!');
+    }
+    console.log('data:')
+    console.log(data);
+}
+function scrapeAPI(url, callbacks = null, type = null){
+    if (typeof url === 'undefined'){
+        console.log('error');
+        return false;
+    }
+    organizrAPI('POST','api/?v1/scrape',{url:url, type:type}).success(function(data) {
+        try {
+            var response = JSON.parse(data);
+        }catch(e) {
+            console.log(e + ' error: ' + data);
+            orgErrorAlert('<h4>' + e + '</h4>' + formatDebug(data));
+            return false;
+        }
+        if(response){
+            if(callbacks){ callbacks.fire(response); }
+        }
+    }).fail(function(xhr) {
+        ajaxloader();
+        console.error("Organizr Function: API Connection Failed");
+    });
+}
 function revokeToken(token,id){
     organizrAPI('POST','api/?v1/token/revoke',{token:token}).success(function(data) {
         try {