parse querystring losslessly (#340)

* parse querystring losslessly

* remove warning

which is hopefully no longer true

* .split doesn't work how I thought it did

* actually detect when a param is missing a =

* use new parsings in python.js

* don't decode query string if it doesn't round-trip

when you percent-decode and re-encode it

* fix tests

* update generators to work with list of lists

* used queryDict in json.js

* fix python and qs parser

* check percent encoding space as + too

fixtures/ansible/get_insecure_full.yml

@@ -1,5 +1,5 @@
 -
-  name: 'https://example.com/'
+  name: 'https://example.com'
   uri:
     url: 'https://example.com'
     method: GET

fixtures/ansible/get_insecure_k.yml

@@ -1,5 +1,5 @@
 -
-  name: 'https://example.com/'
+  name: 'https://example.com'
   uri:
     url: 'https://example.com'
     method: GET

fixtures/ansible/get_insecure_short.yml

@@ -1,5 +1,5 @@
 -
-  name: 'https://www.site.com/'
+  name: 'https://www.site.com'
   uri:
     url: 'https://www.site.com'
     method: GET

fixtures/ansible/post_empty.yml

@@ -1,5 +1,5 @@
 -
-  name: 'http://localhost:28139/'
+  name: 'http://localhost:28139'
   uri:
     url: 'http://localhost:28139'
     method: POST

fixtures/ansible/post_number.yml

@@ -1,5 +1,5 @@
 -
-  name: 'http://a.com/'
+  name: 'http://a.com'
   uri:
     url: 'http://a.com'
     method: POST

fixtures/ansible/put_xput.yml

@@ -1,5 +1,5 @@
 -
-  name: 'http://localhost:9200/twitter/_mapping/user'
+  name: 'http://localhost:9200/twitter/_mapping/user?pretty'
   uri:
     url: 'http://localhost:9200/twitter/_mapping/user?pretty'
     method: PUT

fixtures/dart/get_complex_url_params.dart

@@ -3,7 +3,8 @@ import 'package:http/http.dart' as http;
 void main() async {
   var params = {
     'page': '1',
-    'available': ['', '1'],
+    'available': '',
+    'available': '1',
     'location': '0',
     'city[id]': '0',
     'city[locality]': '',

fixtures/dart/put_xput.dart

@@ -5,14 +5,9 @@ void main() async {
     'Content-Type': 'application/json',
   };
 
-  var params = {
-    'pretty': '',
-  };
-  var query = params.entries.map((p) => '${p.key}=${p.value}').join('&');
-
   var data = '{"properties": {"email": {"type": "keyword"}}}';
 
-  var res = await http.put('http://localhost:9200/twitter/_mapping/user?$query', headers: headers, body: data);
+  var res = await http.put('http://localhost:9200/twitter/_mapping/user?pretty', headers: headers, body: data);
   if (res.statusCode != 200) throw Exception('http.put error: statusCode= ${res.statusCode}');
   print(res.body);
 }

fixtures/elixir/get_complex_url_params.ex

@@ -5,7 +5,8 @@ request = %HTTPoison.Request{
   headers: [],
   params: [
     {~s|page|, ~s|1|},
-    {~s|available|, ["", ~s|1|]},
+    {~s|available|, ""},
+    {~s|available|, ~s|1|},
     {~s|location|, ~s|0|},
     {~s|city[id]|, ~s|0|},
     {~s|city[locality]|, ""},

fixtures/elixir/get_insecure_full.ex

@@ -1,6 +1,6 @@
 request = %HTTPoison.Request{
   method: :get,
-  url: "https://example.com/",
+  url: "https://example.com",
   options: [hackney: [:insecure]],
   headers: [],
   params: [],

fixtures/elixir/get_insecure_k.ex

@@ -1,6 +1,6 @@
 request = %HTTPoison.Request{
   method: :get,
-  url: "https://example.com/",
+  url: "https://example.com",
   options: [hackney: [:insecure]],
   headers: [],
   params: [],

fixtures/elixir/get_insecure_short.ex

@@ -1,6 +1,6 @@
 request = %HTTPoison.Request{
   method: :get,
-  url: "https://www.site.com/",
+  url: "https://www.site.com",
   options: [hackney: [:insecure]],
   headers: [],
   params: [],

fixtures/elixir/get_without_headers.ex

@@ -1,6 +1,6 @@
 request = %HTTPoison.Request{
   method: :get,
-  url: "http://indeed.com/",
+  url: "http://indeed.com",
   options: [],
   headers: [],
   params: [],

fixtures/elixir/post_empty.ex

@@ -1,6 +1,6 @@
 request = %HTTPoison.Request{
   method: :post,
-  url: "http://localhost:28139/",
+  url: "http://localhost:28139",
   options: [],
   headers: [],
   params: [],

fixtures/elixir/post_number.ex

@@ -1,6 +1,6 @@
 request = %HTTPoison.Request{
   method: :post,
-  url: "http://a.com/",
+  url: "http://a.com",
   options: [],
   headers: [],
   params: [],

fixtures/elixir/put_with_T_option.ex

@@ -1,13 +1,11 @@
 request = %HTTPoison.Request{
   method: :put,
-  url: "http://localhost:9200/twitter/_mapping/user",
+  url: "http://localhost:9200/twitter/_mapping/user?pretty",
   options: [],
   headers: [
     {~s|Content-Type|, ~s|application/json|},
   ],
-  params: [
-    {~s|pretty|, ""},
-  ],
+  params: [],
   body: ~s|{"properties": {"email": {"type": "keyword"}}}|
 }
 

fixtures/elixir/put_xput.ex

@@ -1,13 +1,11 @@
 request = %HTTPoison.Request{
   method: :put,
-  url: "http://localhost:9200/twitter/_mapping/user",
+  url: "http://localhost:9200/twitter/_mapping/user?pretty",
   options: [],
   headers: [
     {~s|Content-Type|, ~s|application/json|},
   ],
-  params: [
-    {~s|pretty|, ""},
-  ],
+  params: [],
   body: ~s|{"properties": {"email": {"type": "keyword"}}}|
 }
 

fixtures/matlab/get_charles_syntax.m

@@ -12,10 +12,6 @@ options = weboptions(...
 );
 response = webread(uri, options);
 
-% As there is a query, a full URI may be necessary instead.
-fullURI = 'http://api.ipify.org/?format=json&';
-response = webread(fullURI, options);
-
 %% HTTP Interface
 import matlab.net.*
 import matlab.net.http.*

fixtures/matlab/get_complex_url_params.m

@@ -45,10 +45,6 @@ baseURI = 'https://www.nomador.com/house-sitting/';
 uri = [baseURI '?' char(join(join(params, '='), '&'))];
 response = webread(uri);
 
-% As there is a query, a full URI may be necessary instead.
-fullURI = 'https://www.nomador.com/house-sitting/?page=1&available=&available=1&location=0&city%5Bid%5D=0&city%5Blocality%5D=&city%5Blocality_text%5D=&city%5Badministrative_area_level_2%5D=&city%5Badministrative_area_level_2_text%5D=&city%5Badministrative_area_level_1%5D=&city%5Badministrative_area_level_1_text%5D=&city%5Bcountry%5D=&city%5Bcountry_text%5D=&city%5Blatitude%5D=&city%5Blongitude%5D=&city%5Bzoom%5D=&city%5Bname%5D=&region%5Bid%5D=0&region%5Blocality%5D=&region%5Blocality_text%5D=&region%5Badministrative_area_level_2%5D=&region%5Badministrative_area_level_2_text%5D=&region%5Badministrative_area_level_1%5D=&region%5Badministrative_area_level_1_text%5D=&region%5Bcountry%5D=&region%5Bcountry_text%5D=&region%5Blatitude%5D=&region%5Blongitude%5D=&region%5Bzoom%5D=&region%5Bname%5D=&country=&environment=&population=&period=0&date=2017-03-03&datestart=2017-03-03&dateend=2017-06-24&season=&duration=&isfd=&stopover=';
-response = webread(fullURI);
-
 %% HTTP Interface
 import matlab.net.*
 import matlab.net.http.*

fixtures/matlab/get_insecure_full.m

@@ -5,6 +5,6 @@
 import matlab.net.*
 import matlab.net.http.*
 
-uri = URI('https://example.com/');
+uri = URI('https://example.com');
 options = HTTPOptions('VerifyServerName', false);
 response = RequestMessage().send(uri.EncodedURI, options);

fixtures/matlab/get_user_agent.m

@@ -12,10 +12,6 @@ options = weboptions(...
 );
 response = webread(uri, options);
 
-% As there is a query, a full URI may be necessary instead.
-fullURI = 'http://205.147.98.6/vc/moviesmagic?p=5&pub=testmovie&tkn=817263812';
-response = webread(fullURI, options);
-
 %% HTTP Interface
 import matlab.net.*
 import matlab.net.http.*

fixtures/matlab/get_with_data.m

@@ -9,10 +9,6 @@ uri = [baseURI '?' char(join(join(params, '='), '&'))];
 options = weboptions('HeaderFields', {'X-Api-Key' '123456789'});
 response = webread(uri, options);
 
-% As there is a query, a full URI may be necessary instead.
-fullURI = 'https://synthetics.newrelic.com/synthetics/api/v3/monitors?test=2&limit=100&w=4';
-response = webread(fullURI, options);
-
 %% HTTP Interface
 import matlab.net.*
 import matlab.net.http.*

fixtures/matlab/get_with_data2.m

@@ -12,10 +12,6 @@ options = weboptions(...
 );
 response = webread(uri, options);
 
-% As there is a query, a full URI may be necessary instead.
-fullURI = 'https://api.newrelic.com/v2/alerts_policy_channels.json?policy_id=policy_id&channel_ids=channel_id';
-response = webread(fullURI, options);
-
 %% HTTP Interface
 import matlab.net.*
 import matlab.net.http.*

fixtures/matlab/get_without_headers.m

@@ -6,5 +6,5 @@ response = webread(uri);
 import matlab.net.*
 import matlab.net.http.*
 
-uri = URI('http://indeed.com/');
+uri = URI('http://indeed.com');
 response = RequestMessage().send(uri.EncodedURI);

fixtures/matlab/post_data_binary_with_equals.m

@@ -48,10 +48,6 @@ options = weboptions(...
 );
 response = webwrite(uri, body, options);
 
-% As there is a query, a full URI may be necessary instead.
-fullURI = 'https://localhost/api/service.svc?action=CreateItem&ID=-37&AC=1';
-response = webwrite(fullURI, body, options);
-
 %% HTTP Interface
 import matlab.net.*
 import matlab.net.http.*

fixtures/matlab/post_empty.m

@@ -8,6 +8,6 @@ import matlab.net.*
 import matlab.net.http.*
 import matlab.net.http.io.*
 
-uri = URI('http://localhost:28139/');
+uri = URI('http://localhost:28139');
 body = FileProvider();
 response = RequestMessage('post', [], body).send(uri.EncodedURI);

fixtures/matlab/post_number.m

@@ -8,6 +8,6 @@ import matlab.net.*
 import matlab.net.http.*
 import matlab.net.http.io.*
 
-uri = URI('http://a.com/');
+uri = URI('http://a.com');
 body = JSONProvider(123);
 response = RequestMessage('post', [], body).send(uri.EncodedURI);

fixtures/matlab/put_xput.m

@@ -1,7 +1,5 @@
 %% Web Access using Data Import and Export API
-params = {'pretty' ''};
-baseURI = 'http://localhost:9200/twitter/_mapping/user';
-uri = [baseURI '?' char(join(join(params, '='), '&'))];
+uri = 'http://localhost:9200/twitter/_mapping/user?pretty';
 body = struct(...
     'properties', struct(...
         'email', struct(...
@@ -15,18 +13,13 @@ options = weboptions(...
 );
 response = webwrite(uri, body, options);
 
-% As there is a query, a full URI may be necessary instead.
-fullURI = 'http://localhost:9200/twitter/_mapping/user?pretty';
-response = webwrite(fullURI, body, options);
-
 %% HTTP Interface
 import matlab.net.*
 import matlab.net.http.*
 import matlab.net.http.io.*
 
-params = {'pretty' ''};
 header = HeaderField('Content-Type', 'application/json');
-uri = URI('http://localhost:9200/twitter/_mapping/user', QueryParameter(params'));
+uri = URI('http://localhost:9200/twitter/_mapping/user?pretty');
 body = JSONProvider(struct(...
     'properties', struct(...
         'email', struct(...

fixtures/python/cert_cacert.py

@@ -2,4 +2,4 @@ import requests
 
 cert = '/path/to/the/cert'
 
-response = requests.get('https://example.com/', cert=cert, verify='/path/to/ca-bundle.crt')
+response = requests.get('https://example.com', cert=cert, verify='/path/to/ca-bundle.crt')

fixtures/python/cert_key_capath.py

@@ -2,4 +2,4 @@ import requests
 
 cert = ('/path/to/cert', '/path/to/key')
 
-response = requests.get('https://example.com/', cert=cert, verify='/path/to/ca')
+response = requests.get('https://example.com', cert=cert, verify='/path/to/ca')

fixtures/python/get_charles_syntax.py

@@ -7,13 +7,8 @@ headers = {
     'Accept-Language': 'en-CN;q=1, zh-Hans-CN;q=0.9',
 }
 
-params = [
-    ('format', 'json'),
-]
+params = {
+    'format': 'json',
+}
 
 response = requests.get('http://api.ipify.org/', headers=headers, params=params)
-
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#response = requests.get('http://api.ipify.org/?format=json&', headers=headers)

fixtures/python/get_complex_url_params.py

@@ -1,51 +1,49 @@
 import requests
 
-params = [
-    ('page', '1'),
-    ('available', ['', '1']),
-    ('location', '0'),
-    ('city[id]', '0'),
-    ('city[locality]', ''),
-    ('city[locality_text]', ''),
-    ('city[administrative_area_level_2]', ''),
-    ('city[administrative_area_level_2_text]', ''),
-    ('city[administrative_area_level_1]', ''),
-    ('city[administrative_area_level_1_text]', ''),
-    ('city[country]', ''),
-    ('city[country_text]', ''),
-    ('city[latitude]', ''),
-    ('city[longitude]', ''),
-    ('city[zoom]', ''),
-    ('city[name]', ''),
-    ('region[id]', '0'),
-    ('region[locality]', ''),
-    ('region[locality_text]', ''),
-    ('region[administrative_area_level_2]', ''),
-    ('region[administrative_area_level_2_text]', ''),
-    ('region[administrative_area_level_1]', ''),
-    ('region[administrative_area_level_1_text]', ''),
-    ('region[country]', ''),
-    ('region[country_text]', ''),
-    ('region[latitude]', ''),
-    ('region[longitude]', ''),
-    ('region[zoom]', ''),
-    ('region[name]', ''),
-    ('country', ''),
-    ('environment', ''),
-    ('population', ''),
-    ('period', '0'),
-    ('date', '2017-03-03'),
-    ('datestart', '2017-03-03'),
-    ('dateend', '2017-06-24'),
-    ('season', ''),
-    ('duration', ''),
-    ('isfd', ''),
-    ('stopover', ''),
-]
+params = {
+    'page': '1',
+    'available': [
+        '',
+        '1',
+    ],
+    'location': '0',
+    'city[id]': '0',
+    'city[locality]': '',
+    'city[locality_text]': '',
+    'city[administrative_area_level_2]': '',
+    'city[administrative_area_level_2_text]': '',
+    'city[administrative_area_level_1]': '',
+    'city[administrative_area_level_1_text]': '',
+    'city[country]': '',
+    'city[country_text]': '',
+    'city[latitude]': '',
+    'city[longitude]': '',
+    'city[zoom]': '',
+    'city[name]': '',
+    'region[id]': '0',
+    'region[locality]': '',
+    'region[locality_text]': '',
+    'region[administrative_area_level_2]': '',
+    'region[administrative_area_level_2_text]': '',
+    'region[administrative_area_level_1]': '',
+    'region[administrative_area_level_1_text]': '',
+    'region[country]': '',
+    'region[country_text]': '',
+    'region[latitude]': '',
+    'region[longitude]': '',
+    'region[zoom]': '',
+    'region[name]': '',
+    'country': '',
+    'environment': '',
+    'population': '',
+    'period': '0',
+    'date': '2017-03-03',
+    'datestart': '2017-03-03',
+    'dateend': '2017-06-24',
+    'season': '',
+    'duration': '',
+    'isfd': '',
+    'stopover': '',
+}
 
 response = requests.get('https://www.nomador.com/house-sitting/', params=params)
-
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#response = requests.get('https://www.nomador.com/house-sitting/?page=1&available=&available=1&location=0&city%5Bid%5D=0&city%5Blocality%5D=&city%5Blocality_text%5D=&city%5Badministrative_area_level_2%5D=&city%5Badministrative_area_level_2_text%5D=&city%5Badministrative_area_level_1%5D=&city%5Badministrative_area_level_1_text%5D=&city%5Bcountry%5D=&city%5Bcountry_text%5D=&city%5Blatitude%5D=&city%5Blongitude%5D=&city%5Bzoom%5D=&city%5Bname%5D=&region%5Bid%5D=0&region%5Blocality%5D=&region%5Blocality_text%5D=&region%5Badministrative_area_level_2%5D=&region%5Badministrative_area_level_2_text%5D=&region%5Badministrative_area_level_1%5D=&region%5Badministrative_area_level_1_text%5D=&region%5Bcountry%5D=&region%5Bcountry_text%5D=&region%5Blatitude%5D=&region%5Blongitude%5D=&region%5Bzoom%5D=&region%5Bname%5D=&country=&environment=&population=&period=0&date=2017-03-03&datestart=2017-03-03&dateend=2017-06-24&season=&duration=&isfd=&stopover=')

fixtures/python/get_insecure_full.py

@@ -1,3 +1,3 @@
 import requests
 
-response = requests.get('https://example.com/', verify=False)
+response = requests.get('https://example.com', verify=False)

fixtures/python/get_insecure_k.py

@@ -1,3 +1,3 @@
 import requests
 
-response = requests.get('https://example.com/', verify=False)
+response = requests.get('https://example.com', verify=False)

fixtures/python/get_insecure_short.py

@@ -1,3 +1,3 @@
 import requests
 
-response = requests.get('https://www.site.com/', verify=False)
+response = requests.get('https://www.site.com', verify=False)

fixtures/python/get_user_agent.py

@@ -5,15 +5,10 @@ headers = {
     'User-Agent': 'Mozilla Android6.1',
 }
 
-params = [
-    ('p', '5'),
-    ('pub', 'testmovie'),
-    ('tkn', '817263812'),
-]
+params = {
+    'p': '5',
+    'pub': 'testmovie',
+    'tkn': '817263812',
+}
 
 response = requests.get('http://205.147.98.6/vc/moviesmagic', headers=headers, params=params)
-
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#response = requests.get('http://205.147.98.6/vc/moviesmagic?p=5&pub=testmovie&tkn=817263812', headers=headers)

fixtures/python/get_user_agent_alias.py

@@ -5,15 +5,10 @@ headers = {
     'User-Agent': 'Mozilla Android6.1',
 }
 
-params = [
-    ('p', '5'),
-    ('pub', 'testmovie'),
-    ('tkn', '817263812'),
-]
+params = {
+    'p': '5',
+    'pub': 'testmovie',
+    'tkn': '817263812',
+}
 
 response = requests.get('http://205.147.98.6/vc/moviesmagic', headers=headers, params=params)
-
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#response = requests.get('http://205.147.98.6/vc/moviesmagic?p=5&pub=testmovie&tkn=817263812', headers=headers)

fixtures/python/get_with_data.py

@@ -4,15 +4,10 @@ headers = {
     'X-Api-Key': '123456789',
 }
 
-params = [
-    ('test', '2'),
-    ('limit', '100'),
-    ('w', '4'),
-]
+params = {
+    'test': '2',
+    'limit': '100',
+    'w': '4',
+}
 
 response = requests.get('https://synthetics.newrelic.com/synthetics/api/v3/monitors', headers=headers, params=params)
-
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#response = requests.get('https://synthetics.newrelic.com/synthetics/api/v3/monitors?test=2&limit=100&w=4', headers=headers)

fixtures/python/get_with_data2.py

@@ -5,14 +5,9 @@ headers = {
     'Content-Type': 'application/json',
 }
 
-params = [
-    ('policy_id', 'policy_id'),
-    ('channel_ids', 'channel_id'),
-]
+params = {
+    'policy_id': 'policy_id',
+    'channel_ids': 'channel_id',
+}
 
 response = requests.put('https://api.newrelic.com/v2/alerts_policy_channels.json', headers=headers, params=params)
-
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#response = requests.put('https://api.newrelic.com/v2/alerts_policy_channels.json?policy_id=policy_id&channel_ids=channel_id', headers=headers)

fixtures/python/get_with_env_var.py

@@ -8,13 +8,8 @@ headers = {
     'Authorization': f"Bearer {DO_API_TOKEN}",
 }
 
-params = [
-    ('type', 'distribution'),
-]
+params = {
+    'type': 'distribution',
+}
 
 response = requests.get('https://api.digitalocean.com/v2/images', headers=headers, params=params)
-
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#response = requests.get('https://api.digitalocean.com/v2/images?type=distribution', headers=headers)

fixtures/python/get_without_headers.py

@@ -1,3 +1,3 @@
 import requests
 
-response = requests.get('http://indeed.com/')
+response = requests.get('http://indeed.com')

fixtures/python/multiple_d_post.py

@@ -1,10 +1,5 @@
 import requests
 
-data = {
-    'version': '1.2',
-    'auth_user': 'fdgxf',
-    'auth_pwd': 'oxfdscds',
-    'json_data': '{ "operation": "core/get", "class": "Software", "key": "key" }'
-}
+data = 'version=1.2&auth_user=fdgxf&auth_pwd=oxfdscds&json_data={ "operation": "core/get", "class": "Software", "key": "key" }'
 
 response = requests.post('https://cmdb.litop.local/webservices/rest.php', data=data)

fixtures/python/options.py

@@ -15,18 +15,13 @@ headers = {
     'Access-Control-Request-Headers': 'content-type,csrf',
 }
 
-params = [
-    ('deviceSerialNumber', 'xxx^'),
-    ('deviceType', 'xxx^'),
-    ('guideId', 's56876^'),
-    ('contentType', 'station^'),
-    ('callSign', '^'),
-    ('mediaOwnerCustomerId', 'xxx'),
-]
+params = {
+    'deviceSerialNumber': 'xxx^',
+    'deviceType': 'xxx^',
+    'guideId': 's56876^',
+    'contentType': 'station^',
+    'callSign': '^',
+    'mediaOwnerCustomerId': 'xxx',
+}
 
 response = requests.options('https://layla.amazon.de/api/tunein/queue-and-play', headers=headers, params=params)
-
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#response = requests.options('https://layla.amazon.de/api/tunein/queue-and-play?deviceSerialNumber=xxx^&deviceType=xxx^&guideId=s56876^&contentType=station^&callSign=^&mediaOwnerCustomerId=xxx', headers=headers)

fixtures/python/patch.py

@@ -19,8 +19,8 @@ json_data = {
 
 response = requests.patch('https://ci.example.com/go/api/agents/adb9540a-b954-4571-9d9b-2f330739d4da', headers=headers, json=json_data, auth=('username', 'password'))
 
-# Note: the data is posted as JSON, which might not be serialized by
-# Requests exactly as it appears in the original command. So
+# Note: the data is posted as JSON, which might not be serialized
+# by Requests exactly as it appears in the original command. So
 # the original data is also given.
 #data = '{\n        "hostname": "agent02.example.com",\n        "agent_config_state": "Enabled",\n        "resources": ["Java","Linux"],\n        "environments": ["Dev"]\n        }'
 #response = requests.patch('https://ci.example.com/go/api/agents/adb9540a-b954-4571-9d9b-2f330739d4da', headers=headers, data=data, auth=('username', 'password'))

fixtures/python/post_basic_auth_url_encoded_data.py

@@ -1,7 +1,7 @@
 import requests
 
 data = {
-    'grant_type': 'client_credentials'
+    'grant_type': 'client_credentials',
 }
 
 response = requests.post('http://localhost/api/oauth/token/', data=data, auth=('foo', 'bar'))

fixtures/python/post_data_binary_with_equals.py

@@ -38,17 +38,12 @@ headers = {
     'X-OWA-Attempt': '1',
 }
 
-params = [
-    ('action', 'CreateItem'),
-    ('ID', '-37'),
-    ('AC', '1'),
-]
+params = {
+    'action': 'CreateItem',
+    'ID': '-37',
+    'AC': '1',
+}
 
 data = '{"__type":"CreateItemJsonRequest:#Exchange","Header":{"__type":"JsonRequestHeaders:#Exchange","RequestServerVersion":"Exchange2013","TimeZoneContext":{"__type":"TimeZoneContext:#Exchange","TimeZoneDefinition":{"__type":"TimeZoneDefinitionType:#Exchange","Id":"France Standard Time"}}},"Body":{"__type":"CreateItemRequest:#Exchange","Items":[{"__type":"Message:#Exchange","Subject":"API","Body":{"__type":"BodyContentType:#Exchange","BodyType":"HTML","Value":"<html><head><meta http-equiv=\\"Content-Type\\" content=\\"text/html; charset=UTF-8\\"><style type=\\"text/css\\" style=\\"display:none\\"><!-- p { margin-top: 0px; margin-bottom: 0px; }--></style></head><body dir=\\"ltr\\" style=\\"font-size:12pt;color:#000000;background-color:#FFFFFF;font-family:Calibri,Arial,Helvetica,sans-serif;\\"><p>API Test for NickC<br></p></body></html>"},"Importance":"Normal","From":null,"ToRecipients":[{"Name":"George LUCAS","EmailAddress":"George.LUCAS@nih.mail.edu.fr","RoutingType":"SMTP","MailboxType":"Mailbox","OriginalDisplayName":"George.LUCAS@nih.mail.edu.fr","SipUri":" "}],"CcRecipients":[],"BccRecipients":[],"Sensitivity":"Normal","IsDeliveryReceiptRequested":false,"IsReadReceiptRequested":false}],"ClientSupportsIrm":true,"OutboundCharset":"AutoDetect","MessageDisposition":"SendAndSaveCopy","ComposeOperation":"newMail"}}'
 
 response = requests.post('https://localhost/api/service.svc', headers=headers, params=params, cookies=cookies, data=data)
-
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#response = requests.post('https://localhost/api/service.svc?action=CreateItem&ID=-37&AC=1', headers=headers, cookies=cookies, data=data)

fixtures/python/post_empty.py

@@ -1,3 +1,3 @@
 import requests
 
-response = requests.post('http://localhost:28139/')
+response = requests.post('http://localhost:28139')

fixtures/python/post_escaped_double_quotes_in_single_quotes.py

@@ -1,7 +1,5 @@
 import requests
 
-data = {
-    'foo': '\\"bar\\"'
-}
+data = 'foo=\\"bar\\"'
 
 response = requests.post('http://example.com/', data=data)

fixtures/python/post_escaped_single_quotes_in_double_quotes.py

@@ -1,7 +1,5 @@
 import requests
 
-data = {
-    'foo': '\\\'bar\\\''
-}
+data = 'foo=\\\'bar\\\''
 
 response = requests.post('http://example.com/', data=data)

fixtures/python/post_json_multiple_headers.py

@@ -13,8 +13,8 @@ json_data = {
 
 response = requests.post('https://0.0.0.0/rest/login-sessions', headers=headers, json=json_data, verify=False)
 
-# Note: the data is posted as JSON, which might not be serialized by
-# Requests exactly as it appears in the original command. So
+# Note: the data is posted as JSON, which might not be serialized
+# by Requests exactly as it appears in the original command. So
 # the original data is also given.
 #data = '{"userName":"username123","password":"password123", "authLoginDomain":"local"}'
 #response = requests.post('https://0.0.0.0/rest/login-sessions', headers=headers, data=data, verify=False)

fixtures/python/post_number.py

@@ -2,4 +2,4 @@ import requests
 
 data = '123'
 
-response = requests.post('http://a.com/', data=data)
+response = requests.post('http://a.com', data=data)

fixtures/python/post_quotes_inside_data.py

@@ -1,7 +1,5 @@
 import requests
 
-data = {
-    'field': 'don\'t you like quotes'
-}
+data = 'field=don%27t%20you%20like%20quotes'
 
 response = requests.post('http://google.com', data=data)

fixtures/python/post_same_field_multiple_times.py

@@ -1,9 +1,11 @@
 import requests
 
-data = [
-    ('foo', 'bar'),
-    ('foo', ''),
-    ('foo', 'barbar'),
-]
+data = {
+    'foo': [
+        'bar',
+        '',
+        'barbar',
+    ],
+}
 
 response = requests.post('http://example.com/', data=data)

fixtures/python/post_with_data_raw.py

@@ -1,9 +1,5 @@
 import requests
 
-data = {
-    'msg1': 'wow',
-    'msg2': 'such',
-    'msg3': '@rawmsg'
-}
+data = 'msg1=wow&msg2=such&msg3=@rawmsg'
 
 response = requests.post('http://example.com/post', data=data)

fixtures/python/post_with_data_with_percent_sign.py

@@ -1,7 +1,5 @@
 import requests
 
-data = {
-    'secret': '*%5*!'
-}
+data = 'secret=*%5*!'
 
 response = requests.post('https://postman-echo.com/post', data=data)

fixtures/python/post_with_double_quotes_inside_single_quotes.py

@@ -1,7 +1,5 @@
 import requests
 
-data = {
-    'foo': '"bar"'
-}
+data = 'foo="bar"'
 
 response = requests.post('http://example.com/', data=data)

fixtures/python/post_with_escaped_double_quotes.py

@@ -1,7 +1,5 @@
 import requests
 
-data = {
-    'foo': '"bar"'
-}
+data = 'foo="bar"'
 
 response = requests.post('http://example.com/', data=data)

fixtures/python/post_with_single_quotes_inside_double_quotes.py

@@ -1,7 +1,7 @@
 import requests
 
 data = {
-    'foo': '\'bar\''
+    'foo': '\'bar\'',
 }
 
 response = requests.post('http://example.com/', data=data)

fixtures/python/post_with_urlencoded_data.py

@@ -14,7 +14,7 @@ headers = {
 
 data = {
     'msg1': 'wow',
-    'msg2': 'such'
+    'msg2': 'such',
 }
 
 response = requests.post('http://fiddle.jshell.net/echo/html/', headers=headers, data=data)

fixtures/python/post_with_urlencoded_data_and_headers.py

@@ -32,7 +32,7 @@ data = {
     'Longitude': '-79.4107246398925',
     'Latitude': '43.6552047278685',
     'ZoomLevel': '13',
-    'CurrentPage': '1'
+    'CurrentPage': '1',
 }
 
 response = requests.post('http://www.realtor.ca/api/Listing.svc/PropertySearch_Post', headers=headers, data=data)

fixtures/python/put_with_T_option.py

@@ -4,10 +4,6 @@ headers = {
     'Content-Type': 'application/json',
 }
 
-params = [
-    ('pretty', ''),
-]
-
 json_data = {
     'properties': {
         'email': {
@@ -16,14 +12,10 @@ json_data = {
     },
 }
 
-response = requests.put('http://localhost:9200/twitter/_mapping/user', headers=headers, params=params, json=json_data)
+response = requests.put('http://localhost:9200/twitter/_mapping/user?pretty', headers=headers, json=json_data)
 
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#
-# The data is also posted as JSON, which might not be serialized
-# exactly as it appears in the original command. So the original
-# data is also given.
+# Note: the data is posted as JSON, which might not be serialized
+# by Requests exactly as it appears in the original command. So
+# the original data is also given.
 #data = '{"properties": {"email": {"type": "keyword"}}}'
 #response = requests.put('http://localhost:9200/twitter/_mapping/user?pretty', headers=headers, data=data)

fixtures/python/put_xput.py

@@ -4,10 +4,6 @@ headers = {
     'Content-Type': 'application/json',
 }
 
-params = [
-    ('pretty', ''),
-]
-
 json_data = {
     'properties': {
         'email': {
@@ -16,14 +12,10 @@ json_data = {
     },
 }
 
-response = requests.put('http://localhost:9200/twitter/_mapping/user', headers=headers, params=params, json=json_data)
+response = requests.put('http://localhost:9200/twitter/_mapping/user?pretty', headers=headers, json=json_data)
 
-# Note: original query string below. It seems impossible to parse and
-# reproduce query strings 100% accurately so the one below is given
-# in case the reproduced version is not "correct".
-#
-# The data is also posted as JSON, which might not be serialized
-# exactly as it appears in the original command. So the original
-# data is also given.
+# Note: the data is posted as JSON, which might not be serialized
+# by Requests exactly as it appears in the original command. So
+# the original data is also given.
 #data = '{"properties": {"email": {"type": "keyword"}}}'
 #response = requests.put('http://localhost:9200/twitter/_mapping/user?pretty', headers=headers, data=data)

fixtures/r/get_charles_syntax.r

@@ -12,8 +12,3 @@ params = list(
 )
 
 res <- httr::GET(url = 'http://api.ipify.org/', httr::add_headers(.headers=headers), query = params)
-
-#NB. Original query string below. It seems impossible to parse and
-#reproduce query strings 100% accurately so the one below is given
-#in case the reproduced version is not "correct".
-# res <- httr::GET(url = 'http://api.ipify.org/?format=json&', httr::add_headers(.headers=headers))

fixtures/r/get_complex_url_params.r

@@ -44,8 +44,3 @@ params = list(
 )
 
 res <- httr::GET(url = 'https://www.nomador.com/house-sitting/', query = params)
-
-#NB. Original query string below. It seems impossible to parse and
-#reproduce query strings 100% accurately so the one below is given
-#in case the reproduced version is not "correct".
-# res <- httr::GET(url = 'https://www.nomador.com/house-sitting/?page=1&available=&available=1&location=0&city%5Bid%5D=0&city%5Blocality%5D=&city%5Blocality_text%5D=&city%5Badministrative_area_level_2%5D=&city%5Badministrative_area_level_2_text%5D=&city%5Badministrative_area_level_1%5D=&city%5Badministrative_area_level_1_text%5D=&city%5Bcountry%5D=&city%5Bcountry_text%5D=&city%5Blatitude%5D=&city%5Blongitude%5D=&city%5Bzoom%5D=&city%5Bname%5D=&region%5Bid%5D=0&region%5Blocality%5D=&region%5Blocality_text%5D=&region%5Badministrative_area_level_2%5D=&region%5Badministrative_area_level_2_text%5D=&region%5Badministrative_area_level_1%5D=&region%5Badministrative_area_level_1_text%5D=&region%5Bcountry%5D=&region%5Bcountry_text%5D=&region%5Blatitude%5D=&region%5Blongitude%5D=&region%5Bzoom%5D=&region%5Bname%5D=&country=&environment=&population=&period=0&date=2017-03-03&datestart=2017-03-03&dateend=2017-06-24&season=&duration=&isfd=&stopover=')

fixtures/r/get_insecure_full.r

@@ -1,3 +1,3 @@
 require(httr)
 
-res <- httr::GET(url = 'https://example.com/', config = httr::config(ssl_verifypeer = FALSE))
+res <- httr::GET(url = 'https://example.com', config = httr::config(ssl_verifypeer = FALSE))

fixtures/r/get_insecure_k.r

@@ -1,3 +1,3 @@
 require(httr)
 
-res <- httr::GET(url = 'https://example.com/', config = httr::config(ssl_verifypeer = FALSE))
+res <- httr::GET(url = 'https://example.com', config = httr::config(ssl_verifypeer = FALSE))

fixtures/r/get_insecure_short.r

@@ -1,3 +1,3 @@
 require(httr)
 
-res <- httr::GET(url = 'https://www.site.com/', config = httr::config(ssl_verifypeer = FALSE))
+res <- httr::GET(url = 'https://www.site.com', config = httr::config(ssl_verifypeer = FALSE))

fixtures/r/get_user_agent.r

@@ -12,8 +12,3 @@ params = list(
 )
 
 res <- httr::GET(url = 'http://205.147.98.6/vc/moviesmagic', httr::add_headers(.headers=headers), query = params)
-
-#NB. Original query string below. It seems impossible to parse and
-#reproduce query strings 100% accurately so the one below is given
-#in case the reproduced version is not "correct".
-# res <- httr::GET(url = 'http://205.147.98.6/vc/moviesmagic?p=5&pub=testmovie&tkn=817263812', httr::add_headers(.headers=headers))

fixtures/r/get_without_headers.r

@@ -1,3 +1,3 @@
 require(httr)
 
-res <- httr::GET(url = 'http://indeed.com/')
+res <- httr::GET(url = 'http://indeed.com')

fixtures/r/options.r

@@ -25,8 +25,3 @@ params = list(
 )
 
 res <- httr::OPTIONS(url = 'https://layla.amazon.de/api/tunein/queue-and-play', httr::add_headers(.headers=headers), query = params)
-
-#NB. Original query string below. It seems impossible to parse and
-#reproduce query strings 100% accurately so the one below is given
-#in case the reproduced version is not "correct".
-# res <- httr::OPTIONS(url = 'https://layla.amazon.de/api/tunein/queue-and-play?deviceSerialNumber=xxx^&deviceType=xxx^&guideId=s56876^&contentType=station^&callSign=^&mediaOwnerCustomerId=xxx', httr::add_headers(.headers=headers))

fixtures/r/post_data_binary_with_equals.r

@@ -47,8 +47,3 @@ params = list(
 data = '{"__type":"CreateItemJsonRequest:#Exchange","Header":{"__type":"JsonRequestHeaders:#Exchange","RequestServerVersion":"Exchange2013","TimeZoneContext":{"__type":"TimeZoneContext:#Exchange","TimeZoneDefinition":{"__type":"TimeZoneDefinitionType:#Exchange","Id":"France Standard Time"}}},"Body":{"__type":"CreateItemRequest:#Exchange","Items":[{"__type":"Message:#Exchange","Subject":"API","Body":{"__type":"BodyContentType:#Exchange","BodyType":"HTML","Value":"<html><head><meta http-equiv=\\"Content-Type\\" content=\\"text/html; charset=UTF-8\\"><style type=\\"text/css\\" style=\\"display:none\\"><!-- p { margin-top: 0px; margin-bottom: 0px; }--></style></head><body dir=\\"ltr\\" style=\\"font-size:12pt;color:#000000;background-color:#FFFFFF;font-family:Calibri,Arial,Helvetica,sans-serif;\\"><p>API Test for NickC<br></p></body></html>"},"Importance":"Normal","From":null,"ToRecipients":[{"Name":"George LUCAS","EmailAddress":"George.LUCAS@nih.mail.edu.fr","RoutingType":"SMTP","MailboxType":"Mailbox","OriginalDisplayName":"George.LUCAS@nih.mail.edu.fr","SipUri":" "}],"CcRecipients":[],"BccRecipients":[],"Sensitivity":"Normal","IsDeliveryReceiptRequested":false,"IsReadReceiptRequested":false}],"ClientSupportsIrm":true,"OutboundCharset":"AutoDetect","MessageDisposition":"SendAndSaveCopy","ComposeOperation":"newMail"}}'
 
 res <- httr::POST(url = 'https://localhost/api/service.svc', httr::add_headers(.headers=headers), query = params, httr::set_cookies(.cookies = cookies), body = data)
-
-#NB. Original query string below. It seems impossible to parse and
-#reproduce query strings 100% accurately so the one below is given
-#in case the reproduced version is not "correct".
-# res <- httr::POST(url = 'https://localhost/api/service.svc?action=CreateItem&ID=-37&AC=1', httr::add_headers(.headers=headers), httr::set_cookies(.cookies = cookies), body = data)

fixtures/r/post_empty.r

@@ -1,3 +1,3 @@
 require(httr)
 
-res <- httr::POST(url = 'http://localhost:28139/')
+res <- httr::POST(url = 'http://localhost:28139')

fixtures/r/post_number.r

@@ -2,4 +2,4 @@ require(httr)
 
 data = '123'
 
-res <- httr::POST(url = 'http://a.com/', body = data)
+res <- httr::POST(url = 'http://a.com', body = data)

fixtures/r/put_xput.r

@@ -4,15 +4,6 @@ headers = c(
   `Content-Type` = 'application/json'
 )
 
-params = list(
-  `pretty` = ''
-)
-
 data = '{"properties": {"email": {"type": "keyword"}}}'
 
-res <- httr::PUT(url = 'http://localhost:9200/twitter/_mapping/user', httr::add_headers(.headers=headers), query = params, body = data)
-
-#NB. Original query string below. It seems impossible to parse and
-#reproduce query strings 100% accurately so the one below is given
-#in case the reproduced version is not "correct".
-# res <- httr::PUT(url = 'http://localhost:9200/twitter/_mapping/user?pretty', httr::add_headers(.headers=headers), body = data)
+res <- httr::PUT(url = 'http://localhost:9200/twitter/_mapping/user?pretty', httr::add_headers(.headers=headers), body = data)

fixtures/strest/get_complex_url_params.strest.yml

@@ -10,9 +10,10 @@ requests:
           value: '1'
         -
           name: available
-          value:
-            - ""
-            - '1'
+          value: ""
+        -
+          name: available
+          value: '1'
         -
           name: location
           value: '0'

fixtures/strest/get_insecure_full.strest.yml

@@ -3,5 +3,5 @@ allowInsecure: true
 requests:
   curl_converter:
     request:
-      url: 'https://example.com/'
+      url: 'https://example.com'
       method: GET

fixtures/strest/get_insecure_k.strest.yml

@@ -3,5 +3,5 @@ allowInsecure: true
 requests:
   curl_converter:
     request:
-      url: 'https://example.com/'
+      url: 'https://example.com'
       method: GET

fixtures/strest/get_insecure_short.strest.yml

@@ -3,5 +3,5 @@ allowInsecure: true
 requests:
   curl_converter:
     request:
-      url: 'https://www.site.com/'
+      url: 'https://www.site.com'
       method: GET

fixtures/strest/post_empty.strest.yml

@@ -2,5 +2,5 @@ version: 2
 requests:
   curl_converter:
     request:
-      url: 'http://localhost:28139/'
+      url: 'http://localhost:28139'
       method: POST

fixtures/strest/post_number.strest.yml

@@ -2,7 +2,7 @@ version: 2
 requests:
   curl_converter:
     request:
-      url: 'http://a.com/'
+      url: 'http://a.com'
       method: POST
       postData:
         mimeType: application/json

fixtures/strest/put_xput.strest.yml

@@ -2,7 +2,7 @@ version: 2
 requests:
   curl_converter:
     request:
-      url: 'http://localhost:9200/twitter/_mapping/user'
+      url: 'http://localhost:9200/twitter/_mapping/user?pretty'
       method: PUT
       postData:
         mimeType: application/json
@@ -14,7 +14,3 @@ requests:
         -
           name: Content-Type
           value: application/json
-      queryString:
-        -
-          name: pretty
-          value: ""

generators/dart.js

@@ -54,15 +54,10 @@ export const _toDart = r => {
 
   const hasQuery = r.query
   if (hasQuery) {
+    // TODO: dict won't work with repeated keys
     s += '  var params = {\n'
-    for (const paramName in r.query) {
-      const rawValue = r.query[paramName]
-      let paramValue
-      if (Array.isArray(rawValue)) {
-        paramValue = '[' + rawValue.map(repr).join(', ') + ']'
-      } else {
-        paramValue = repr(rawValue)
-      }
+    for (const [paramName, rawValue] of r.query) {
+      const paramValue = repr(rawValue === null ? '' : rawValue)
       s += '    ' + repr(paramName) + ': ' + paramValue + ',\n'
     }
     s += '  };\n'

generators/elixir.js

@@ -66,15 +66,8 @@ function getQueryDict (request) {
     return '[]'
   }
   let queryDict = '[\n'
-  for (const paramName in request.query) {
-    const rawValue = request.query[paramName]
-    let paramValue
-    if (Array.isArray(rawValue)) {
-      paramValue = '[' + rawValue.map(repr).join(', ') + ']'
-    } else {
-      paramValue = repr(rawValue)
-    }
-    queryDict += `    {${repr(paramName)}, ${paramValue}},\n`
+  for (const [paramName, rawValue] of request.query) {
+    queryDict += `    {${repr(paramName)}, ${repr(rawValue)}},\n`
   }
   queryDict += '  ]'
   return queryDict

generators/json.js

@@ -16,22 +16,6 @@ function repr (value, isKey) {
   return isKey ? "'" + jsesc(value, { quotes: 'single' }) + "'" : value
 }
 
-function getQueries (request) {
-  const queries = {}
-  for (const paramName in request.query) {
-    const rawValue = request.query[paramName]
-    let paramValue
-    if (Array.isArray(rawValue)) {
-      paramValue = rawValue.map(repr)
-    } else {
-      paramValue = repr(rawValue)
-    }
-    queries[repr(paramName)] = paramValue
-  }
-
-  return queries
-}
-
 function getDataString (request) {
   /*
     if ( !request.isDataRaw && request.data.startsWith('@') ) {
@@ -96,8 +80,6 @@ function getFilesString (request) {
 }
 
 export const _toJsonString = request => {
-  const requestJson = {}
-
   // curl automatically prepends 'http' if the scheme is missing, but python fails and returns an error
   // we tack it on here to mimic curl
   if (!request.url.match(/https?:/)) {
@@ -107,9 +89,16 @@ export const _toJsonString = request => {
     request.urlWithoutQuery = 'http://' + request.urlWithoutQuery
   }
 
-  requestJson.url = request.urlWithoutQuery.replace(/\/$/, '')
-  requestJson.raw_url = request.url
-  requestJson.method = request.method
+  const requestJson = {
+    url: (request.queryDict ? request.urlWithoutQuery : request.url).replace(/\/$/, ''),
+    // url: request.queryDict ? request.urlWithoutQuery : request.url,
+    raw_url: request.url,
+    // TODO: move this after .query?
+    method: request.method
+  }
+  // if (request.queryDict) {
+  //   requestJson.query = request.queryDict
+  // }
 
   if (request.cookies) {
     const cookies = {}
@@ -129,8 +118,9 @@ export const _toJsonString = request => {
     requestJson.headers = headers
   }
 
-  if (request.query) {
-    requestJson.queries = getQueries(request)
+  if (request.queryDict) {
+    // TODO: rename
+    requestJson.queries = request.queryDict
   }
 
   if (request.data && typeof request.data === 'string') {

generators/matlab/common.js

@@ -143,8 +143,8 @@ const containsBody = (request) => {
 
 const prepareQueryString = (request) => {
   let response = null
-  if (request.query) {
-    const params = addCellArray(request.query, [], '', 1)
+  if (request.queryDict) {
+    const params = addCellArray(request.queryDict, [], '', 1)
     response = setVariableValue('params', params)
   }
   return response

generators/matlab/httpinterface.js

@@ -58,9 +58,12 @@ const prepareHeaders = (request) => {
 }
 
 const prepareURI = (request) => {
-  const uriParams = [repr(request.urlWithoutQuery)]
-  if (request.query) {
+  const uriParams = []
+  if (request.queryDict) {
+    uriParams.push(repr(request.urlWithoutQuery))
     uriParams.push('QueryParameter(params\')')
+  } else {
+    uriParams.push(repr(request.url))
   }
   return callFunction('uri', 'URI', uriParams)
 }

generators/matlab/webservices.js

@@ -110,7 +110,7 @@ const prepareOptions = (request, options) => {
 
 const prepareBasicURI = (request) => {
   const response = []
-  if (request.query) {
+  if (request.queryDict) {
     response.push(setVariableValue('baseURI', repr(request.urlWithoutQuery)))
     response.push(setVariableValue('uri', `[baseURI '?' ${paramsString}]`))
   } else {
@@ -163,14 +163,6 @@ const prepareWebCall = (request, options) => {
   }
   lines.push(callFunction('response', webFunction, params))
 
-  if (request.query) {
-    params[0] = 'fullURI'
-    lines.push('',
-      '% As there is a query, a full URI may be necessary instead.',
-      setVariableValue('fullURI', repr(request.url)),
-      callFunction('response', webFunction, params)
-    )
-  }
   return lines
 }
 

generators/python.js

@@ -1,7 +1,6 @@
 import * as util from '../util.js'
 
 import jsesc from 'jsesc'
-import querystring from 'query-string'
 
 function reprWithVariable (value, hasEnvironmentVariable) {
   if (!value) {
@@ -20,23 +19,7 @@ function repr (value) {
   return reprWithVariable(value, false)
 }
 
-function getQueryDict (request) {
-  let queryDict = 'params = [\n'
-  for (const paramName in request.query) {
-    const rawValue = request.query[paramName]
-    let paramValue
-    if (Array.isArray(rawValue)) {
-      paramValue = '[' + rawValue.map(repr).join(', ') + ']'
-    } else {
-      paramValue = repr(rawValue)
-    }
-    queryDict += '    (' + repr(paramName) + ', ' + paramValue + '),\n'
-  }
-  queryDict += ']\n'
-  return queryDict
-}
-
-function jsonToPython (obj, indent = 0) {
+function objToPython (obj, indent = 0) {
   let s = ''
   switch (typeof obj) {
     case 'string':
@@ -57,7 +40,7 @@ function jsonToPython (obj, indent = 0) {
         } else {
           s += '[\n'
           for (const item of obj) {
-            s += ' '.repeat(indent + 4) + jsonToPython(item, indent + 4) + ',\n'
+            s += ' '.repeat(indent + 4) + objToPython(item, indent + 4) + ',\n'
           }
           s += ' '.repeat(indent) + ']'
         }
@@ -69,7 +52,7 @@ function jsonToPython (obj, indent = 0) {
           s += '{\n'
           for (const [k, v] of Object.entries(obj)) {
             // repr() because JSON keys must be strings.
-            s += ' '.repeat(indent + 4) + repr(k) + ': ' + jsonToPython(v, indent + 4) + ',\n'
+            s += ' '.repeat(indent + 4) + repr(k) + ': ' + objToPython(v, indent + 4) + ',\n'
           }
           s += ' '.repeat(indent) + '}'
         }
@@ -81,6 +64,21 @@ function jsonToPython (obj, indent = 0) {
   return s
 }
 
+function objToDictOrListOfTuples (obj) {
+  if (!Array.isArray(obj)) {
+    return objToPython(obj)
+  }
+  if (obj.length === 0) {
+    return '[]'
+  }
+  let s = '[\n'
+  for (const vals of obj) {
+    s += '    (' + vals.map(objToPython).join(', ') + '),\n'
+  }
+  s += ']'
+  return s
+}
+
 function getDataString (request) {
   if (!request.isDataRaw && request.data.startsWith('@')) {
     const filePath = request.data.slice(1)
@@ -91,84 +89,41 @@ function getDataString (request) {
     }
   }
 
-  const parsedQueryString = querystring.parse(request.data, { sort: false })
-  const keyCount = Object.keys(parsedQueryString).length
-  const singleKeyOnly = keyCount === 1 && !parsedQueryString[Object.keys(parsedQueryString)[0]]
-  const singularData = request.isDataBinary || singleKeyOnly
-  if (singularData) {
-    const dataString = 'data = ' + repr(request.data) + '\n'
-    const isJson = request.headers &&
-          (request.headers['Content-Type'] === 'application/json' ||
-           request.headers['content-type'] === 'application/json')
-    if (isJson) {
-      try {
-        const dataAsJson = JSON.parse(request.data)
-        // TODO: we actually want to know how it's serialized by
-        // simplejson or Python's builtin json library,
-        // which is what Requests uses
-        // https://github.com/psf/requests/blob/b0e025ade7ed30ed53ab61f542779af7e024932e/requests/models.py#L473
-        // but this is hopefully good enough.
-        const roundtrips = JSON.stringify(dataAsJson) === request.data
-        const jsonDataString = 'json_data = ' + jsonToPython(dataAsJson) + '\n'
-        // Remove "Content-Type" from the headers dict
-        // because Requests adds it automatically when you use json=
-        if (roundtrips) {
-          delete request.headers['Content-Type']
-          delete request.headers['content-type']
-          if (Object.keys(request.headers).length === 0) {
-            delete request.headers
-          }
+  const dataString = 'data = ' + repr(request.data) + '\n'
+
+  const isJson = request.headers &&
+        (request.headers['Content-Type'] === 'application/json' ||
+          request.headers['content-type'] === 'application/json')
+  if (isJson) {
+    try {
+      const dataAsJson = JSON.parse(request.data)
+      // We actually want to know how it's serialized by simplejson or
+      // Python's builtin json library, which is what Requests uses
+      // but this is hopefully good enough.
+      const roundtrips = JSON.stringify(dataAsJson) === request.data
+      const jsonDataString = 'json_data = ' + objToPython(dataAsJson) + '\n'
+      // Remove "Content-Type" from the headers dict
+      // because Requests adds it automatically when you use json=
+      if (roundtrips) {
+        delete request.headers['Content-Type']
+        delete request.headers['content-type']
+        if (Object.keys(request.headers).length === 0) {
+          delete request.headers
         }
-        return [dataString, jsonDataString, roundtrips]
-      } catch {}
-    }
-    return [dataString, null, null]
-  } else {
-    return [getMultipleDataString(request, parsedQueryString), null, null]
-  }
-}
-
-function getMultipleDataString (request, parsedQueryString) {
-  let repeatedKey = false
-  for (const key in parsedQueryString) {
-    const value = parsedQueryString[key]
-    if (Array.isArray(value)) {
-      repeatedKey = true
-    }
-  }
-
-  let dataString
-  if (repeatedKey) {
-    dataString = 'data = [\n'
-    for (const key in parsedQueryString) {
-      const value = parsedQueryString[key]
-      if (Array.isArray(value)) {
-        for (let i = 0; i < value.length; i++) {
-          dataString += '    (' + repr(key) + ', ' + repr(value[i]) + '),\n'
-        }
-      } else {
-        dataString += '    (' + repr(key) + ', ' + repr(value) + '),\n'
       }
-    }
-    dataString += ']\n'
-  } else {
-    dataString = 'data = {\n'
-    const elementCount = Object.keys(parsedQueryString).length
-    let i = 0
-    for (const key in parsedQueryString) {
-      const value = parsedQueryString[key]
-      dataString += '    ' + repr(key) + ': ' + repr(value)
-      if (i === elementCount - 1) {
-        dataString += '\n'
-      } else {
-        dataString += ',\n'
-      }
-      ++i
-    }
-    dataString += '}\n'
+      return [dataString, jsonDataString, roundtrips]
+    } catch {}
   }
 
-  return dataString
+  const [parsedQuery, parsedQueryAsDict] = util.parseQueryString(request.data)
+  if (!request.isDataBinary &&
+      parsedQuery &&
+      parsedQuery.length &&
+      !parsedQuery.some(p => p[1] === null)) {
+    const dataPythonObj = 'data = ' + objToDictOrListOfTuples(parsedQueryAsDict || parsedQuery) + '\n'
+    return [dataPythonObj, null, null]
+  }
+  return [dataString, null, null]
 }
 
 function getFilesString (request) {
@@ -291,9 +246,9 @@ export const _toPython = request => {
     certStr += '\n'
   }
 
-  let queryDict
+  let queryStr
   if (request.query) {
-    queryDict = getQueryDict(request)
+    queryStr = 'params = ' + objToDictOrListOfTuples(request.queryDict || request.query) + '\n'
   }
 
   let dataString
@@ -326,14 +281,14 @@ export const _toPython = request => {
 
   // curl automatically prepends 'http' if the scheme is missing, but python fails and returns an error
   // we tack it on here to mimic curl
+  // TODO: warn users about unsupported schemes
   if (!request.url.match(/https?:/)) {
     request.url = 'http://' + request.url
   }
   if (!request.urlWithoutQuery.match(/https?:/)) {
     request.urlWithoutQuery = 'http://' + request.urlWithoutQuery
   }
-  let requestLineWithUrlParams = 'response = requests.' + request.method + "('" + request.urlWithoutQuery + "'"
-  let requestLineWithOriginalUrl = 'response = requests.' + request.method + "('" + request.url + "'"
+  let requestLine = 'response = requests.' + request.method + '(' + repr(request.urlWithoutQuery)
 
   let requestLineBody = ''
   if (request.headers) {
@@ -371,8 +326,7 @@ export const _toPython = request => {
   }
   requestLineBody += ')'
 
-  requestLineWithOriginalUrl += requestLineBody.replace(', params=params', '')
-  requestLineWithUrlParams += requestLineBody
+  requestLine += requestLineBody
 
   let pythonCode = ''
 
@@ -398,8 +352,8 @@ export const _toPython = request => {
   if (headerDict) {
     pythonCode += headerDict + '\n'
   }
-  if (queryDict) {
-    pythonCode += queryDict + '\n'
+  if (queryStr) {
+    pythonCode += queryStr + '\n'
   }
   if (certStr) {
     pythonCode += certStr + '\n'
@@ -411,32 +365,15 @@ export const _toPython = request => {
   } else if (filesString) {
     pythonCode += filesString + '\n'
   }
-  pythonCode += requestLineWithUrlParams
+  pythonCode += requestLine
 
-  if (request.query && jsonDataString && !jsonDataStringRoundtrips) {
+  if (jsonDataString && !jsonDataStringRoundtrips) {
     pythonCode += '\n\n' +
-            '# Note: original query string below. It seems impossible to parse and\n' +
-            '# reproduce query strings 100% accurately so the one below is given\n' +
-            '# in case the reproduced version is not "correct".\n' +
-            '#\n' +
-            '# The data is also posted as JSON, which might not be serialized\n' +
-            '# exactly as it appears in the original command. So the original\n' +
-            '# data is also given.\n'
-    pythonCode += '#' + dataString
-    pythonCode += '#' + requestLineWithOriginalUrl.replace(', json=json_data', ', data=data')
-  } else if (request.query) {
-    pythonCode += '\n\n' +
-            '# Note: original query string below. It seems impossible to parse and\n' +
-            '# reproduce query strings 100% accurately so the one below is given\n' +
-            '# in case the reproduced version is not "correct".\n'
-    pythonCode += '#' + requestLineWithOriginalUrl
-  } else if (jsonDataString && !jsonDataStringRoundtrips) {
-    pythonCode += '\n\n' +
-            '# Note: the data is posted as JSON, which might not be serialized by\n' +
-            '# Requests exactly as it appears in the original command. So\n' +
+            '# Note: the data is posted as JSON, which might not be serialized\n' +
+            '# by Requests exactly as it appears in the original command. So\n' +
             '# the original data is also given.\n'
     pythonCode += '#' + dataString
-    pythonCode += '#' + requestLineWithUrlParams.replace(', json=json_data', ', data=data')
+    pythonCode += '#' + requestLine.replace(', json=json_data', ', data=data')
   }
 
   return pythonCode + '\n'

generators/r.js

@@ -24,8 +24,8 @@ function repr (value) {
 
 function getQueryDict (request) {
   let queryDict = 'params = list(\n'
-  queryDict += Object.keys(request.query).map((paramName) => {
-    const rawValue = request.query[paramName]
+  queryDict += Object.keys(request.queryDict).map((paramName) => {
+    const rawValue = request.queryDict[paramName]
     let paramValue
     if (Array.isArray(rawValue)) {
       paramValue = 'c(' + rawValue.map(repr).join(', ') + ')'
@@ -131,7 +131,7 @@ export const _toR = request => {
   }
 
   let queryDict
-  if (request.query) {
+  if (request.queryDict) {
     queryDict = getQueryDict(request)
   }
 
@@ -150,14 +150,14 @@ export const _toR = request => {
   if (!request.urlWithoutQuery.match(/https?:/)) {
     request.urlWithoutQuery = 'http://' + request.urlWithoutQuery
   }
-  let requestLineWithUrlParams = 'res <- httr::' + request.method.toUpperCase() + '(url = \'' + request.urlWithoutQuery + '\''
-  let requestLineWithOriginalUrl = 'res <- httr::' + request.method.toUpperCase() + '(url = \'' + request.url + '\''
+  const url = request.queryDict ? request.urlWithoutQuery : request.url
+  let requestLine = 'res <- httr::' + request.method.toUpperCase() + '(url = \'' + url + '\''
 
   let requestLineBody = ''
   if (request.headers) {
     requestLineBody += ', httr::add_headers(.headers=headers)'
   }
-  if (request.query) {
+  if (request.queryDict) {
     requestLineBody += ', query = params'
   }
   if (request.cookies) {
@@ -179,8 +179,7 @@ export const _toR = request => {
   }
   requestLineBody += ')'
 
-  requestLineWithOriginalUrl += requestLineBody.replace(', query = params', '')
-  requestLineWithUrlParams += requestLineBody
+  requestLine += requestLineBody
 
   let rstatsCode = ''
   rstatsCode += 'require(httr)\n\n'
@@ -198,15 +197,7 @@ export const _toR = request => {
   } else if (filesString) {
     rstatsCode += filesString + '\n'
   }
-  rstatsCode += requestLineWithUrlParams
-
-  if (request.query) {
-    rstatsCode += '\n\n' +
-            '#NB. Original query string below. It seems impossible to parse and\n' +
-            '#reproduce query strings 100% accurately so the one below is given\n' +
-            '#in case the reproduced version is not "correct".\n'
-    rstatsCode += '# ' + requestLineWithOriginalUrl
-  }
+  rstatsCode += requestLine
 
   return rstatsCode + '\n'
 }

generators/strest.js

@@ -35,12 +35,8 @@ function getDataString (request) {
 }
 
 function getQueryList (request) {
-  const queryList = []
-  for (const paramName in request.query) {
-    const rawValue = request.query[paramName]
-    queryList.push({ name: paramName, value: rawValue })
-  }
-  return queryList
+  // Convert nulls to empty string
+  return request.query.map(p => ({ name: p[0], value: p[1] || '' }))
 }
 
 export const _toStrest = request => {

test.js

@@ -13,7 +13,6 @@ import { fixturesDir, converters } from './test-utils.js'
 // language-specific directories: node/, php/, python/, parser/
 // we get a list of all input files, iterate over it, and if an
 // output file exists, compare the output.
-
 const curlCommandsDir = path.resolve(fixturesDir, 'curl_commands')
 
 const testArgs = yargs(hideBin(process.argv))
@@ -67,7 +66,8 @@ for (const fileName of testFileNames) {
     const converter = converters[outputLanguage]
 
     const filePath = path.resolve(fixturesDir, outputLanguage, fileName.replace(/\.sh$/, converter.extension))
-    const testName = converter.name + ': ' + fileName.replace(/_/g, ' ').replace(/\.sh$/, '')
+    const testName = fileName.replace(/_/g, ' ').replace(/\.sh$/, '')
+    const fullTestName = converter.name + ': ' + testName
 
     if (fs.existsSync(filePath)) {
       // normalize code for just \n line endings (aka fix input under Windows)
@@ -76,19 +76,18 @@ for (const fileName of testFileNames) {
       try {
         actual = converter.converter(inputFileContents)
       } catch (e) {
-        console.error('could not convert' + testName + ' to ' + outputLanguage)
-        console.error()
+        console.error('Failed converting ' + fileName + ' to ' + converter.name + ':')
         console.error(e)
         process.exit(1)
       }
       if (outputLanguage === 'parser') {
-        test(testName, t => {
+        test(fullTestName, t => {
           // TODO: `actual` is a needless roundtrip
           t.deepEquals(JSON.parse(actual), JSON.parse(expected))
           t.end()
         })
       } else {
-        test(testName, t => {
+        test(fullTestName, t => {
           t.equal(actual, expected)
           t.end()
         })

util.js

@@ -2,7 +2,6 @@ import URL from 'url'
 
 import cookie from 'cookie'
 import nunjucks from 'nunjucks'
-import querystring from 'query-string'
 
 import parser from './parser.js'
 
@@ -859,6 +858,78 @@ const parseArgs = (args, opts) => {
   return parsedArguments
 }
 
+export const parseQueryString = (s) => {
+  // if url is 'example.com?' => s is ''
+  // if url is 'example.com'  => s is null
+  if (!s) {
+    return [null, null]
+  }
+
+  const asList = []
+  for (const param of s.split('&')) {
+    const [key, val] = param.split(/=(.*)/s, 2)
+    let decodedKey
+    let decodedVal
+    try {
+      decodedKey = decodeURIComponent(key)
+      decodedVal = val === undefined ? null : decodeURIComponent(val)
+    } catch (e) {
+      if (e instanceof URIError) {
+        // Query string contains invalid percent encoded characters,
+        // we cannot properly convert it.
+        return [null, null]
+      }
+      throw e
+    }
+    try {
+      // If the query string doesn't round-trip, we cannot properly convert it.
+      // TODO: this is too strict. Ideally we want to check how each runtime/library
+      // percent encodes query strings. For example, a %27 character in the input query
+      // string will be decoded to a ' but won't be re-encoded into a %27 by encodeURIComponent
+      const roundTripKey = encodeURIComponent(decodedKey)
+      const roundTripVal = encodeURIComponent(decodedVal)
+      if ((roundTripKey !== key && roundTripKey.replace('%20', '+') !== key) ||
+          (decodedVal && (roundTripVal !== val && roundTripVal.replace('%20', '+') !== val))) {
+        return [null, null]
+      }
+    } catch (e) {
+      if (e instanceof URIError) {
+        return [null, null]
+      }
+      throw e
+    }
+    asList.push([decodedKey, decodedVal])
+  }
+
+  // Group keys
+  const asDict = {}
+  let prevKey = null
+  for (const [key, val] of asList) {
+    if (prevKey === key) {
+      asDict[key].push(val)
+    } else {
+      if (!has(asDict, key)) {
+        asDict[key] = [val]
+      } else {
+        // If there's a repeated key with a different key between
+        // one of its repetitions, there is no way to represent
+        // this query string as a dictionary.
+        return [asList, null]
+      }
+    }
+    prevKey = key
+  }
+
+  // Convert lists with 1 element to the element
+  for (const [key, val] of Object.entries(asDict)) {
+    if (val.length === 1) {
+      asDict[key] = val[0]
+    }
+  }
+
+  return [asList, asDict]
+}
+
 const buildRequest = parsedArguments => {
   // TODO: handle multiple URLs
   if (!parsedArguments.url || !parsedArguments.url.length) {
@@ -939,6 +1010,7 @@ const buildRequest = parsedArguments => {
   const urlObject = URL.parse(url) // eslint-disable-line
   // if GET request with data, convert data to query string
   // NB: the -G flag does not change the http verb. It just moves the data into the url.
+  // TODO: this probably has a lot of mismatches with curl
   if (parsedArguments.get) {
     urlObject.query = urlObject.query ? urlObject.query : ''
     if (has(parsedArguments, 'data')) {
@@ -952,6 +1024,7 @@ const buildRequest = parsedArguments => {
 
       urlQueryString += parsedArguments.data.join('&')
       urlObject.query += urlQueryString
+      // TODO: url and urlObject will be different if url has an #id
       url += urlQueryString
       delete parsedArguments.data
     }
@@ -959,25 +1032,28 @@ const buildRequest = parsedArguments => {
   if (urlObject.query && urlObject.query.endsWith('&')) {
     urlObject.query = urlObject.query.slice(0, -1)
   }
-  const query = querystring.parse(urlObject.query, { sort: false })
-  for (const param in query) {
-    if (query[param] === null) {
-      query[param] = ''
+  const [queryAsList, queryAsDict] = parseQueryString(urlObject.query)
+  // Most software libraries don't let you distinguish between a=&b= and a&b,
+  // so if we get an `a&b`-type query string, don't bother.
+  const request = { url }
+  if (!queryAsList || queryAsList.some((p) => p[1] === null)) {
+    request.urlWithoutQuery = url // TODO: rename?
+  } else {
+    urlObject.search = null // Clean out the search/query portion.
+    request.urlWithoutQuery = URL.format(urlObject)
+
+    if (queryAsList.length > 0) {
+      request.query = queryAsList
+      if (queryAsDict) {
+        request.queryDict = queryAsDict
+      }
     }
   }
 
-  urlObject.search = null // Clean out the search/query portion.
-  const request = {
-    url,
-    urlWithoutQuery: URL.format(urlObject)
-  }
   if (parsedArguments.compressed) {
     request.compressed = true
   }
 
-  if (Object.keys(query).length > 0) {
-    request.query = query
-  }
   if (headers) {
     request.headers = headers
   }