Always add scheme to URL (#397)

src/generators/elixir.ts

@@ -232,14 +232,7 @@ export const _toElixir = (
   warnings?: Warnings
 ): [string, Warnings] => {
   warnings = warnings || [];
-  // curl automatically prepends 'http' if the scheme is missing, but python fails and returns an error
-  // we tack it on here to mimic curl
-  if (!request.url.match(/https?:/)) {
-    request.url = "http://" + request.url;
-  }
-  if (!request.urlWithoutQuery.match(/https?:/)) {
-    request.urlWithoutQuery = "http://" + request.urlWithoutQuery;
-  }
+
   if (request.cookies) {
     util.deleteHeader(request, "cookie");
   }

src/generators/json.ts

@@ -125,14 +125,6 @@ export const _toJsonString = (
   warnings?: Warnings
 ): [string, Warnings] => {
   warnings = warnings || [];
-  // curl automatically prepends 'http' if the scheme is missing, but python fails and returns an error
-  // we tack it on here to mimic curl
-  if (!request.url.match(/https?:/)) {
-    request.url = "http://" + request.url;
-  }
-  if (!request.urlWithoutQuery.match(/https?:/)) {
-    request.urlWithoutQuery = "http://" + request.urlWithoutQuery;
-  }
 
   const requestJson: JSONOutput = {
     url: (request.queryDict ? request.urlWithoutQuery : request.url).replace(

src/generators/python.ts

@@ -864,16 +864,6 @@ export const _toPython = (
     headerDict += "}\n";
   }
 
-  // curl automatically prepends 'http' if the scheme is missing, but python fails and returns an error
-  // we tack it on here to mimic curl
-  // TODO: warn users about unsupported schemes
-  if (!request.url.match(/https?:/)) {
-    request.url = "http://" + request.url;
-  }
-  if (!request.urlWithoutQuery.match(/https?:/)) {
-    request.urlWithoutQuery = "http://" + request.urlWithoutQuery;
-  }
-
   let requestLine;
   if (
     ["GET", "HEAD", "POST", "PATCH", "PUT", "DELETE", "OPTIONS"].includes(

src/generators/r.ts

@@ -159,14 +159,6 @@ export const _toR = (
   } else if (request.multipartUploads) {
     filesString = getFilesString(request);
   }
-  // curl automatically prepends 'http' if the scheme is missing, but rstats fails and returns an error
-  // we tack it on here to mimic curl
-  if (!request.url.match(/https?:/)) {
-    request.url = "http://" + request.url;
-  }
-  if (!request.urlWithoutQuery.match(/https?:/)) {
-    request.urlWithoutQuery = "http://" + request.urlWithoutQuery;
-  }
   const url = request.queryDict ? request.urlWithoutQuery : request.url;
 
   let requestLine = "res <- httr::";

src/generators/strest.ts

@@ -101,9 +101,6 @@ export const _toStrest = (
   if (request.insecure) {
     response.allowInsecure = true;
   }
-  if (!request.urlWithoutQuery.match(/https?:/)) {
-    request.urlWithoutQuery = "http://" + request.urlWithoutQuery;
-  }
   response.requests = {
     curl_converter: {
       request: {

src/util.ts

@@ -1451,6 +1451,27 @@ function buildRequest(
     method = "POST";
   }
 
+  // curl automatically prepends 'http' if the scheme is missing,
+  // but many libraries fail if your URL doesn't have it,
+  // we tack it on here to mimic curl
+  //
+  // RFC 3986 3.1 says
+  //   scheme      = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
+  // but curl will accept a digit/plus/minus/dot in the first character
+  // curl will also accept a url with one / like http:/localhost
+  const schemeMatch = url.match(/^([a-zA-Z0-9+-.]*):\/\/?/);
+  if (schemeMatch) {
+    const scheme = schemeMatch[1].toLowerCase();
+    if (scheme !== "http" && scheme !== "https") {
+      warnings.push(["bad-scheme", `Protocol "${scheme}" not supported`]);
+    }
+    url = scheme + "://" + url.slice(schemeMatch[0].length);
+  } else {
+    // curl's default scheme is actually https://
+    // but we don't do that because, unlike curl, most libraries won't downgrade to http if you ask for https
+    url = "http://" + url;
+  }
+
   let urlObject = URL.parse(url); // eslint-disable-line
   if (parsedArguments["upload-file"]) {
     // TODO: it's more complicated

test/fixtures/ansible/strange_http_method.yml

@@ -1,6 +1,6 @@
 -
-  name: 'localhost:28139'
+  name: 'http://localhost:28139'
   uri:
-    url: 'localhost:28139'
+    url: 'http://localhost:28139'
     method: WHAT
   register: result

test/fixtures/cfml/delete_content_type.cfm

@@ -1,8 +0,0 @@
-httpService = new http();
-httpService.setUrl("localhost:28139");
-httpService.setMethod("POST");
-httpService.addParam(type="header", name="Content-Type", value="application/json");
-httpService.addParam(type="body", value="{}");
-
-result = httpService.send().getPrefix();
-writeDump(result);

test/fixtures/cfml/post_quotes_inside_data.cfm

@@ -1,5 +1,5 @@
 httpService = new http();
-httpService.setUrl("localhost:28139");
+httpService.setUrl("http://localhost:28139");
 httpService.setMethod("POST");
 httpService.addParam(type="header", name="Content-Type", value="application/x-www-form-urlencoded");
 httpService.addParam(type="body", value="field=don%27t%20you%20like%20quotes");

test/fixtures/curl_commands/delete_content_type.sh

@@ -1 +0,0 @@
-curl --data '{}' -H 'Content-Type: application/json' localhost:28139

test/fixtures/dart/strange_http_method.dart

@@ -1,7 +1,7 @@
 import 'package:http/http.dart' as http;
 
 void main() async {
-  var url = Uri.parse('localhost:28139');
+  var url = Uri.parse('http://localhost:28139');
   var res = await http.what(url);
   if (res.statusCode != 200) throw Exception('http.what error: statusCode= ${res.statusCode}');
   print(res.body);

test/fixtures/go/strange_http_method.go

@@ -9,7 +9,7 @@ import (
 
 func main() {
 	client := &http.Client{}
-	req, err := http.NewRequest("wHat", "localhost:28139", nil)
+	req, err := http.NewRequest("wHat", "http://localhost:28139", nil)
 	if err != nil {
 		log.Fatal(err)
 	}

test/fixtures/java/get_silent.java

@@ -7,7 +7,7 @@ import java.util.Scanner;
 class Main {
 
 	public static void main(String[] args) throws IOException {
-		URL url = new URL("localhost:28139");
+		URL url = new URL("http://localhost:28139");
 		HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
 		httpConn.setRequestMethod("GET");
 

test/fixtures/java/get_url_starting_with_http.java

@@ -7,7 +7,7 @@ import java.util.Scanner;
 class Main {
 
 	public static void main(String[] args) throws IOException {
-		URL url = new URL("httpbin.org/test");
+		URL url = new URL("http://httpbin.org/test");
 		HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
 		httpConn.setRequestMethod("GET");
 

test/fixtures/java/get_with_header_without_value.java

@@ -7,7 +7,7 @@ import java.util.Scanner;
 class Main {
 
 	public static void main(String[] args) throws IOException {
-		URL url = new URL("localhost:28139/get");
+		URL url = new URL("http://localhost:28139/get");
 		HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
 		httpConn.setRequestMethod("GET");
 

test/fixtures/java/head_without_http.java

@@ -7,7 +7,7 @@ import java.util.Scanner;
 class Main {
 
 	public static void main(String[] args) throws IOException {
-		URL url = new URL("localhost:28139/page");
+		URL url = new URL("http://localhost:28139/page");
 		HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
 		httpConn.setRequestMethod("HEAD");
 

test/fixtures/java/post_quotes_inside_data.java

@@ -8,7 +8,7 @@ import java.util.Scanner;
 class Main {
 
 	public static void main(String[] args) throws IOException {
-		URL url = new URL("localhost:28139");
+		URL url = new URL("http://localhost:28139");
 		HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
 		httpConn.setRequestMethod("POST");
 

test/fixtures/java/strange_http_method.java

@@ -7,7 +7,7 @@ import java.util.Scanner;
 class Main {
 
 	public static void main(String[] args) throws IOException {
-		URL url = new URL("localhost:28139");
+		URL url = new URL("http://localhost:28139");
 		HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
 		httpConn.setRequestMethod("wHat");
 

test/fixtures/javascript/delete_content_type.js

@@ -1,7 +0,0 @@
-fetch('localhost:28139', {
-    method: 'POST',
-    headers: {
-        'Content-Type': 'application/json'
-    },
-    body: JSON.stringify({})
-});

test/fixtures/javascript/get_with_header_without_value.js

@@ -1,4 +1,4 @@
-fetch('localhost:28139/get', {
+fetch('http://localhost:28139/get', {
     headers: {
         'Content-Type': 'text/xml;charset=UTF-8',
         'getWorkOrderCancel': ''

test/fixtures/javascript/post_quotes_inside_data.js

@@ -1,4 +1,4 @@
-fetch('localhost:28139', {
+fetch('http://localhost:28139', {
     method: 'POST',
     body: new URLSearchParams({
         'field': 'don\'t you like quotes'

test/fixtures/javascript/strange_http_method.js

@@ -1,3 +1,3 @@
-fetch('localhost:28139', {
+fetch('http://localhost:28139', {
     method: 'wHat'
 });

test/fixtures/matlab/strange_http_method.m

@@ -5,5 +5,5 @@
 import matlab.net.*
 import matlab.net.http.*
 
-uri = URI('localhost:28139');
+uri = URI('http://localhost:28139');
 response = RequestMessage('what').send(uri.EncodedURI);

test/fixtures/node-axios/delete_content_type.js

@@ -1,11 +0,0 @@
-const axios = require('axios');
-
-const response = await axios.post(
-    'localhost:28139',
-    {},
-    {
-        headers: {
-            'Content-Type': 'application/json'
-        }
-    }
-);

test/fixtures/node-axios/get_url_starting_with_http.js

@@ -1,3 +1,3 @@
 const axios = require('axios');
 
-const response = await axios.get('httpbin.org/test');
+const response = await axios.get('http://httpbin.org/test');

test/fixtures/node-axios/get_with_header_without_value.js

@@ -1,6 +1,6 @@
 const axios = require('axios');
 
-const response = await axios.get('localhost:28139/get', {
+const response = await axios.get('http://localhost:28139/get', {
     headers: {
         'Content-Type': 'text/xml;charset=UTF-8',
         'getWorkOrderCancel': ''

test/fixtures/node-axios/head_without_http.js

@@ -1,3 +1,3 @@
 const axios = require('axios');
 
-const response = await axios.head('localhost:28139/page');
+const response = await axios.head('http://localhost:28139/page');

test/fixtures/node-axios/post_quotes_inside_data.js

@@ -1,7 +1,7 @@
 const axios = require('axios');
 
 const response = await axios.post(
-    'localhost:28139',
+    'http://localhost:28139',
     new URLSearchParams({
         'field': 'don\'t you like quotes'
     })

test/fixtures/node-axios/strange_http_method.js

@@ -1,3 +1,3 @@
 const axios = require('axios');
 
-const response = await axios('localhost:28139');
+const response = await axios('http://localhost:28139');

test/fixtures/node-request/strange_http_method.js

@@ -1,7 +1,7 @@
 var request = require('request');
 
 var options = {
-    url: 'localhost:28139',
+    url: 'http://localhost:28139',
     method: 'WHAT'
 };
 

test/fixtures/node/get_with_header_without_value.js

@@ -1,6 +1,6 @@
 import fetch from 'node-fetch';
 
-fetch('localhost:28139/get', {
+fetch('http://localhost:28139/get', {
     headers: {
         'Content-Type': 'text/xml;charset=UTF-8',
         'getWorkOrderCancel': ''

test/fixtures/node/post_quotes_inside_data.js

@@ -1,6 +1,6 @@
 import fetch from 'node-fetch';
 
-fetch('localhost:28139', {
+fetch('http://localhost:28139', {
     method: 'POST',
     body: new URLSearchParams({
         'field': 'don\'t you like quotes'

test/fixtures/node/strange_http_method.js

@@ -1,5 +1,5 @@
 import fetch from 'node-fetch';
 
-fetch('localhost:28139', {
+fetch('http://localhost:28139', {
     method: 'wHat'
 });

test/fixtures/parser/strange_http_method.json

@@ -1,5 +1,5 @@
 {
-  "url": "localhost:28139",
+  "url": "http://localhost:28139",
   "method": "wHat",
-  "urlWithoutQuery": "localhost:28139"
+  "urlWithoutQuery": "http://localhost:28139"
 }

test/fixtures/php/delete_content_type.php

@@ -1,13 +0,0 @@
-<?php
-$ch = curl_init();
-curl_setopt($ch, CURLOPT_URL, 'localhost:28139');
-curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
-curl_setopt($ch, CURLOPT_HTTPHEADER, [
-    'Content-Type' => 'application/json',
-]);
-curl_setopt($ch, CURLOPT_POSTFIELDS, '{}');
-
-$response = curl_exec($ch);
-
-curl_close($ch);

test/fixtures/php/post_quotes_inside_data.php

@@ -1,6 +1,6 @@
 <?php
 $ch = curl_init();
-curl_setopt($ch, CURLOPT_URL, 'localhost:28139');
+curl_setopt($ch, CURLOPT_URL, 'http://localhost:28139');
 curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
 curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
 curl_setopt($ch, CURLOPT_HTTPHEADER, [

test/fixtures/php/strange_http_method.php

@@ -1,6 +1,6 @@
 <?php
 $ch = curl_init();
-curl_setopt($ch, CURLOPT_URL, 'localhost:28139');
+curl_setopt($ch, CURLOPT_URL, 'http://localhost:28139');
 curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
 curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'wHat');
 

test/fixtures/python/delete_content_type.py

@@ -1,10 +0,0 @@
-import requests
-
-headers = {
-    # Already added when you pass json=
-    # 'Content-Type': 'application/json',
-}
-
-json_data = {}
-
-response = requests.post('http://localhost:28139', headers=headers, json=json_data)

test/fixtures/rust/strange_http_method.rs

@@ -2,7 +2,7 @@ extern crate reqwest;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let res = reqwest::Client::new()
-        .what("localhost:28139")
+        .what("http://localhost:28139")
         .send()?
         .text()?;
     println!("{}", res);