Redirect-chain checking in bulk with Google Sheets

/**
 * Processes a given URL to extract and return data in a format compatible with Google Sheets.
 * It captures HTTP headers, HTML meta tags, and logs any remarks during processing.
 * 
 * @param {string} url - The URL to process, or an empty string if no URL is provided.
 * @return {Array} - Results formatted for Google Sheets.
 */
function scraper(url) {
  let result = [];
  // Initialize objects to store HTTP header, HTML meta tags, and log information
  let http = initializeHttpObject(url);
  let html = initializeHtmlObject();
  let logs = { remarks: [] };

  // If no URL is provided, only return column headers
  if (!url) {
    result.push(Object.keys(http).concat(Object.keys(html), Object.keys(logs)));
    return result;
  }

  // Fetch the URL and reverse the order of results for processing
  let output = fetchUrl(http.requestedUrl, http.chainLength);
  output = output.reverse();

  // Process each item in the fetch response
  for (let i = 0; i < output.length; i++) {
    let call = output[i].call;
    // Process individual call if it exists
    if (call) {
      processCall(call, http, html, logs);
    }
  }

  // Prepare the final result after processing all calls
  finalizeHttpObject(http, logs);
  result.push(Object.values(http).concat(Object.values(html), Object.values(logs)));
  return result;
}

// Initialize HTTP object with default values for each property
function initializeHttpObject(url) {
  return {
    requestedUrl: url,
    requestedUrlStatus: false,
    loadedUrl: false,
    loadedUrlStatus: false,
    fullChain: [],
    allLocations: [],
    allStatusCodes: [],
    chainLength: 0,
    httpxrobots: []
  };
}

// Initialize HTML object with default false or null values
function initializeHtmlObject() {
  return {
    title: false,
    robots: false,
    canonical: false,
    canonicalSameAsLoadedUrl: null
  };
}

// Processes each call (response) in the fetch output
function processCall(call, http, html, logs) {
  // Process URL redirection and status
  if (call.location) {
    http.fullChain.push(`${call.url} => ${call.status}, note: to a relative location: ${call.location}`);
  } else {
    http.fullChain.push(`${call.url} => ${call.status}`);
  }
  // Accumulate all status codes and update chain length for redirects
  http.allStatusCodes.push(call.status);
  if (call.status > 300 && call.status < 399) {
    http.chainLength += 1;
  }
  // Store each URL and HTTP x-robots tag
  http.allLocations.push(call.url);
  http.httpxrobots.push(call.httpxrobots);

  // Process HTML tags if HTML content is available
  if (call.html) {
    processHtmlTags(call.html, html, call.url);
  }

  // Store any remarks from processing
  if (call.remarks) {
    logs.remarks.push(call.remarks);
  }
}

// Extracts and sets HTML tag information from the HTML parser
function processHtmlTags(htmlParser, htmlObject, url) {
  htmlObject.title = htmlParser('title').first().text() || false;
  htmlObject.robots = htmlParser('meta[name=robots]').attr('content') || false;
  htmlObject.canonical = htmlParser('link[rel=canonical]').attr('href') || false;
  htmlObject.canonicalSameAsLoadedUrl = (htmlObject.canonical === url);
}

// Finalizes the HTTP object by compiling and formatting data
function finalizeHttpObject(http, logs) {
  // Set the status of requested and final loaded URLs
  http.requestedUrlStatus = http.allStatusCodes[0];
  http.loadedUrl = http.allLocations[http.allLocations.length - 1];
  http.loadedUrlStatus = http.allStatusCodes[http.allStatusCodes.length - 1];

  // Log error if the final loaded URL results in an error
  if (http.loadedUrlStatus > 399) {
    logs.remarks.push(`URL error: final loaded URL results in an error, status code: ${http.loadedUrlStatus}`);
  }

  // Concatenate various arrays into newline-separated strings for output
  http.httpxrobots = http.httpxrobots.join('\n');
  http.allLocations = http.allLocations.join("\n");
  http.allStatusCodes = http.allStatusCodes.join("\n > ");
  http.fullChain = http.fullChain.join("\n");
  logs.remarks = logs.remarks.join("\n").replace("\n\n","\n");
}

/**
 * Fetches a given URL and returns the response.
 * Handles HTTP requests, follows redirects up to a specified limit, and parses the response.
 * 
 * @param {string} url - The URL to fetch.
 * @param {number} chainLength - The current length of the redirect chain.
 * @return {Object} - The response from the fetch operation.
 */
function fetchUrl(url, chainLength) {
  let result = [];
  const chainLimit = 10; // Maximum number of redirects to follow
  const chainWarning = 3; // Number of redirects after which to issue a warning

  try {
    let options = configureRequestOptions();

    // Fetch URL if provided
    if (url) {
      let response = UrlFetchApp.fetch(url, options);
      let row = {
        url: url,
        status: response.getResponseCode(),
        httpxrobots: response.getHeaders()['X-Robots-Tag'] || '',
        location: response.getHeaders()['Location'] || '',
        html: response.getContentText() ? Cheerio.load(response.getContentText()) : null,
        remarks: ''
      };

      // Handle URL redirection
      if (row.location && chainLength < chainLimit) {
        let correctedLocation = parseUrlForRedirect(row.location, url);
        row.location = correctedLocation.fullUrl;
        result = result.concat(fetchUrl(correctedLocation.fullUrl, chainLength + 1));
      }

      // Add warning or limit remarks based on redirect chain length
      if (chainLength === chainWarning) {
        row.remarks += `Warning: Redirect chain length has reached ${chainWarning}.`;
      }
      if (chainLength >= chainLimit) {
        row.remarks += `Limit reached: Stopped following redirects at chain length ${chainLimit}.`;
      }

      // Add the row to the result
      result.push({ call: row });
    }
  } catch (error) {
    // Push error information to the result
    result.push({ error: error.toString() });
  }
  return result;
}

/**
 * Configures and returns options for the HTTP request.
 * 
 * @return {Object} - Object containing configured request options.
 */
function configureRequestOptions() {
  let options = {
    'muteHttpExceptions': true,
    'followRedirects': false // Manually handle redirects
  };
  // Additional configuration can be added here (e.g., headers, authorization)
  return options;
}

/**
 * Parses a URL for redirection, handling relative URLs.
 * Constructs a full URL if a relative path is provided.
 * 
 * @param {string} location - The location header from the response.
 * @param {string} currentUrl - The current URL being processed.
 * @return {Object} - Object containing the full URL for redirection.
 */
function parseUrlForRedirect(location, currentUrl) {
  // Handle relative URLs by constructing a full URL
  if (location.startsWith('/')) {
    let { protocol, host } = parseUrl(currentUrl);
    return { fullUrl: `${protocol}://${host}${location}` };
  }
  return { fullUrl: location };
}

/**
 * Parses a given URL and returns its components.
 * Components include protocol, host, port, path, query, and fragment.
 * 
 * @param {string} url - The URL to parse.
 * @return {Object} - An object containing the components of the URL.
 */
function parseUrl(url) {
  let match = url.match(/^(http|https|ftp)?(?:[:\/]*)([a-z0-9.-]*)(?::([0-9]+))?(\/[^?#]*)?(?:\?([^#]*))?(?:#(.*))?$/i);
  let components = {
    protocol: match[1] || '',
    host: match[2] || '',
    port: match[3] || '',
    path: match[4] || '',
    query: match[5] || '',
    fragment: match[6] || ''
  };
  return components;
}

    // uncomment if you want to enter an area that is protected with basic auth
    /**
     * const basicAuthUser = "user";
     * const basicAuthPass = "pass";
     * options.headers = {
     *  "Authorization": "Basic " + Utilities.base64Encode(basicAuthUser + ":" + basicAuthPass, Utilities.Charset.UTF_8),
     * }
     */
    

    // uncomment if you want to enter an area that is protected with an authorization token
    /**
     * const token = "XXXXXXXXXXXXXXXXXXXXX";
     * options.contentType: 'application/json',
     * options.headers = {
     *  "Authorization" : token 
     * }
     */

    // uncomment if you just need to send custom headers. I'll give you an example
    /** 
     * options.headers = {
     *  "custom_header_key": "a-key",
     *  "custom_header_secret": "a-secret"
     * }
    */