Menu
I'm trying to use parts of this Python script (taken from here) in a Rust program I'm writing. How can I construct a reqwest request with the same content?
def login(login_url, username, password=None, token=None):
"""Log in to Kattis.
At least one of password or token needs to be provided.
Returns a requests.Response with cookies needed to be able to submit
"""
login_args = {'user': username, 'script': 'true'}
if password:
login_args['password'] = password
if token:
login_args['token'] = token
response = requests.post(login_url, data=login_args, headers=_HEADERS)
return response
def submit(submit_url, cookies, problem, language, files, mainclass='', tag=''):
"""Make a submission.
The url_opener argument is an OpenerDirector object to use (as
returned by the login() function)
Returns the requests.Result from the submission
"""
data = {'submit': 'true',
'submit_ctr': 2,
'language': language,
'mainclass': mainclass,
'problem': problem,
'tag': tag,
'script': 'true'}
sub_files = []
for f in files:
with open(f) as sub_file:
sub_files.append(('sub_file[]',
(os.path.basename(f),
sub_file.read(),
'application/octet-stream')))
return requests.post(submit_url, data=data, files=sub_files, cookies=cookies, headers=_HEADERS)
(check out the link above for the rest of the code)
Currently I've got this (I'm not sure if cookies are handled)
let config = get_config().await?;
let mut default_headers = header::HeaderMap::new();
default_headers.insert(
header::USER_AGENT,
header::HeaderValue::from_static("kattis-cli-submit"),
);
let client = reqwest::ClientBuilder::new()
.default_headers(default_headers)
.cookie_store(true)
.build()?;
// Login
let login_map = serde_json::json!({
"user": config.username.as_str(),
"script": "true",
"token": config.token.as_str(),
});
let login_response = client
.post(&config.login_url)
.header("Content-Type", "application/x-www-form-urlencoded")
.json(&login_map)
.send()
.await?;
println!("{:?}", login_response);
// Make a submission
let submission_map = serde_json::json!({
"submit": "true",
"submit_ctr": "2",
"language": language,
"mainclass": problem,
"problem": problem,
"script": "true",
});
println!("{}", &submission_map);
let mut form = multipart::Form::new();
let mut sub_file = multipart::Part::text(submission).file_name(submission_filename);
sub_file = sub_file.mime_str("application/octet-stream").unwrap();
form = form.part("sub_file[]", sub_file);
let submission_response = client
.post(&config.submit_url)
.json(&submission_map)
.multipart(form)
// .build();
.send()
.await?
.text()
.await?;
let config = get_config().await?;
let mut default_headers = header::HeaderMap::new();
default_headers.insert(
header::USER_AGENT,
header::HeaderValue::from_static("kattis-cli-submit"),
);
let client = reqwest::ClientBuilder::new()
.default_headers(default_headers)
.cookie_store(true)
.build()?;
// Login
let login_map = serde_json::json!({
"user": config.username.as_str(),
"script": "true",
"token": config.token.as_str(),
});
let login_response = client
.post(&config.login_url)
.header("Content-Type", "application/x-www-form-urlencoded")
.json(&login_map)
.send()
.await?;
println!("{:?}", login_response);
// Make a submission
let submission_map = serde_json::json!({
"submit": "true",
"submit_ctr": "2",
"language": language,
"mainclass": problem,
"problem": problem,
"script": "true",
});
println!("{}", &submission_map);
let mut form = multipart::Form::new();
let mut sub_file = multipart::Part::text(submission).file_name(submission_filename);
sub_file = sub_file.mime_str("application/octet-stream").unwrap();
form = form.part("sub_file[]", sub_file);
let submission_response = client
.post(&config.submit_url)
.json(&submission_map)
.multipart(form)
// .build();
.send()
.await?
.text()
.await?;
println!("Submission response:\n{:?}", submission_response);
Which for reference spits out
{"user": {"username": Some("[username]"), "token": Some("[token]")}, "kattis": {"loginurl": Some("https://open.kattis.com/login"), "hostname": Some("open.kattis.com"), "submissionurl": Some("https://open.kattis.com/submit"), "submissionsurl": Some("https://open.kattis.com/submissions")}}
Response { url: "https://open.kattis.com/login", status: 200, headers: {"date": "Sun, 13 Sep 2020 14:19:15 GMT", "content-type": "text/html; charset=UTF-8", "transfer-encoding": "chunked", "connection": "keep-alive", "set-cookie": "__cfduid=d0417cc7406c8d91b8659327fff8d5d9a1600006752; expires=Tue, 13-Oct-20 14:19:12 GMT; path=/; domain=.kattis.com; HttpOnly; SameSite=Lax", "set-cookie": "EduSiteCookie=75f873b9-5442-45be-b442-be08f349e09c; path=/; domain=.kattis.com; secure; HttpOnly", "expires": "Thu, 19 Nov 1981 08:52:00 GMT", "cache-control": "no-store, no-cache, must-revalidate", "pragma": "no-cache", "cf-cache-status": "DYNAMIC", "cf-request-id": "05296ea065000015fc7ca80200000001", "expect-ct": "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"", "server": "cloudflare", "cf-ray": "5d22807a39b015fc-ARN", "alt-svc": "h3-27=\":443\"; ma=86400, h3-28=\":443\"; ma=86400, h3-29=\":443\"; ma=86400"} }
{"language":"C++","mainclass":"ants","problem":"ants","script":"true","submit":"true","submit_ctr":"2"}
Submission response:
"<!DOCTYPE html>\n\n\n<html lang=\"en\">\n<head>\n <meta charset=\"UTF-8\" >\n <meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n <title>Log in or sign up for Kattis – Kattis, Kattis</title>\n\n <link href=\"//ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/themes/base/jquery-ui.min.css\" rel=\"stylesheet\">\n\n <script src=\"//ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js\"></script>\n <script src=\"//ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js\"></script>\n\n <!-- Fonts/Icons -->\n <link href=\"//cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css\" rel=\"stylesheet\">\n\n <link href=\"//fonts.googleapis.com/css?family=Open+Sans:400,300,300italic,400italic,600,600italic,700,800,700italic,800italic%7CMerriweather:400,400italic,700\" rel=\"stylesheet\" type=\"text/css\">\n\n <!-- Bootstrap CSS -->\n <link href=\"//cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/css/bootstrap.min.css\" rel=\"stylesheet\">\n\n <!-- Bootstrap datetimepicker CSS-->\n <link href=\"//cdnjs.cloudflare.com/ajax/libs/bootstrap-datetimepicker/4.17.47/css/bootstrap-datetimepicker.min.css\" rel=\"stylesheet\">\n\n <!-- DateRangePicker CSS -->\n <link href=\"//cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.css\" rel=\"stylesheet\">\n\n <!-- Editable and Select2 -->\n <link href=\"//cdnjs.cloudflare.com/ajax/libs/select2/3.5.4/select2.css\" rel=\"stylesheet\">\n\n <link rel=\"shortcut icon\" href=\"/favicon\" />\n\n <!-- Own CSS -->\n <link rel=\"stylesheet\" href=\"/css/system.css?03bf93=\">\n <style type=\"text/css\">\n .header {\n background-color: rgb(240,176,52);\n }\n .header .main-nav > ul > li.current:before {\n border-bottom-color: rgb(240,176,52);\n }\n\n div.page-content.clearfix.above-everything.alert.alert-danger { color: #31708f; background: #d9edf7; border-color: #bce8f1; }\r\ndiv.page-content.clearfix.above-everything.alert.alert-danger div.main-content { padding-bottom: 0; }\r\n\n </style>\n\n <script type=\"text/javascript\">\n window.page_loaded_at = new Date();\n jQuery.noConflict();\n </script>\n\n <script type=\"text/javascript\">\n jQuery.ns = function (namespace) {\n var parts = namespace.split(\'.\');\n var last = window;\n for (var i = 0; i < parts.length; i++) {\n last = last[parts[i]] || (last[parts[i]] = {});\n }\n return last;\n };\n</script>\n <script>\njQuery.extend(jQuery.ns(\'Kattis.error\'), (function () {\n var messages = {\"INTERNAL_SERVER_ERROR\":\"Internal server error.\",\"ACCESS_DENIED\":\"Access denied.\",\"NOT_AUTHENTICATED\":\"Not authenticated.\",\"METHOD_NOT_ALLOWED\":\"Method not allowed.\",\"INVALID_JSON\":\"JSON cannot be decoded or encoded data is deeper than the recursion limit.\",\"BAD_CSRF_TOKEN\":\"Token does not match session\'s csrf_token\",\"SESSION_NAME_EMPTY\":\"Session\'s name must be non empty.\",\"SESSION_START_TIME_EMPTY\":\"Session\'s start time must be non empty.\",\"SESSION_START_TIME_PASSED\":\"Session\'s start time has already passed.\",\"SESSION_DURATION_EMPTY\":\"Session\'s duration must be non empty.\",\"SESSION_DURATION_NEGATIVE\":\"Session\'s duration must be a positive number.\",\"SESSION_DURATION_EXCEEDED\":\"Maximum duration for the session was exceeded.\",\"SESSION_ALREADY_STARTED\":\"The session has already started.\",\"SESSION_ALREADY_FINISHED\":\"The session is already finished.\",\"USER_CREATED_SESSION_DURATION_EXCEEDED\":\"Contest cannot be longer than 168 hours.\",\"INVALID_PROBLEM_SCORE\":\"Invalid problem score.\",\"INVALID_SESSION_SHORTNAME\":\"Invalid shortname for the session.\",\"INVALID_SESSION_CUTOFF\":\"Invalid cutoff for the session.\",\"INVALID_USER_NAME\":\"Invalid username or email.\",\"SESSION_NOT_FOUND\":\"No such session.\",\"COURSE_NOT_FOUND\":\"No such course.\",\"OFFERING_NOT_FOUND\":\"No such offering.\",\"TEACHER_NOT_FOUND\":\"No such teacher.\",\"TEACHER_CANNOT_REMOVE_SELF\":\"You may not remove yourself as a teacher unless you are an administrator.\",\"AUTHOR_NOT_FOUND\":\"No such author.\",\"JUDGE_NOT_FOUND\":\"No such judge.\",\"JUDGE_ALREADY_EXIST\":\"The user is already a judge.\",\"TEACHER_ALREADY_EXIST\":\"The user is already a teacher.\",\"PROBLEM_NOT_FOUND\":\"No such problem.\",\"TEAM_NOT_FOUND\":\"No such team.\",\"SESSION_PROBLEM_ALREADY_EXIST\":\"The problem has been already added to the session.\",\"SESSION_PROBLEM_DOES_NOT_EXIST\":\"The problem does not relate to the session.\",\"PROBLEM_INDEX_NEGATIVE\":\"Problem index must be non negative.\",\"AUTHOR_IS_CURRENT_TEAM_MEMBER\":\"The user you tried to add is already a member of the current team.\",\"AUTHOR_IS_ANOTHER_TEAM_MEMBER\":\"The user you tried to add is already a member of another team in the current session.\",\"AUTHOR_IS_JUDGE\":\"The user you tried to add is a judge.\",\"AUTHOR_IS_NOT_TEAM_MEMBER\":\"The user you tried to remove is not a team member.\",\"JUDGE_IS_TEAM_MEMBER\":\"The user you tried to add is a session team member or invitee.\",\"SESSION_PUBLISHING_DENIED\":\"You do not have permission to publish this session.\",\"CANNOT_PUBLISH_HISTORICAL_SESSION\":\"You cannot publish a session with a historical start time.\",\"INVALID_TEAM_NAME_TOO_LONG\":\"The team name you are trying to add is too long\",\"TEAM_NAME_IS_NOT_VISIBLE\":\"The team name you are trying to add is not visible\"};\n\n return {\n get_msg: function (error_code) {\n return messages[error_code];\n },\n\n show_msg: function (base_message, error_code) {\n if (error_code) {\n alert(base_message + \": \" + this.get_msg(error_code));\n } else {\n alert(base_message);\n }\n },\n\n show_xhr_msg: function (elem, jqXHR) {\n var base_message = elem.data(\'fail-msg\');\n var code = jqXHR.responseJSON && jqXHR.responseJSON.error &&\n jqXHR.responseJSON.error.code;\n this.show_msg(base_message, code);\n }\n }\n})());\n</script>\n\n \n\n <script type=\"text/javascript\">\nvar rumMOKey=\"a854f3a6dd7ee5e3b7d1641570b79c34\";\n(function(){\nif(window.performance && window.performance.timing && window.performance.navigation) {\n\tvar site24x7_rum_beacon=document.createElement(\'script\');\n\tsite24x7_rum_beacon.async=true;\n\tsite24x7_rum_beacon.setAttribute(\'src\',\'//static.site24x7rum.eu/beacon/site24x7rum-min.js?appKey=\'+rumMOKey);\n\tdocument.getElementsByTagName(\'head\')[0].appendChild(site24x7_rum_beacon);\n}\n})(window)\n</script>\n\n \n</head>\n\n<body class=\"page-master-layout \">\n\n\n<div id=\"wrapper\">\n <header class=\"header\">\n <div class=\"background\">\n \n <div class=\"wrap\">\n <div class=\"fl\">\n <a href=\"/\"><img class=\"logo logo-open\" src=\"/images/site-logo\" alt=\"\" /></a>\n <div class=\"title-wrapper\">\n <div class=\"header-title\">Kattis</div>\n <nav class=\"main-nav\">\n <ul>\n \n <li class=\"\"><a href=\"/problems\">Problems</a></li>\n \n <li class=\"\"><a href=\"/contests\">Contests</a></li>\n \n <li class=\"\"><a href=\"/ranklist\">Ranklists</a></li>\n \n <li class=\"\"><a href=\"/jobs\">Jobs</a></li>\n \n <li class=\"\"><a href=\"/help\">Help</a></li>\n \n </ul>\n </nav>\n </div>\n </div>\n <div class=\"user-side fr\">\n\n <nav class=\"user-nav\">\n <ul class=\"user-nav-ul\">\n <li>\n <form action=\"/search\" class=\"site-search\" method=\"GET\">\n <input type=\"text\" name=\"q\" placeholder=\"Search Kattis\" />\n <a href=\"#\">\n <i class=\"fa fa-search\"></i>\n </a>\n </form>\n </li>\n \n <li><a class=\"btn dark-bg\" href=\"/login\">Log in</a></li>\n </ul>\n\n </nav>\n\n </div>\n </div>\n </div>\n</header>\n\n <!--[if IE]> <div class=\"alert alert-warning\" role=\"alert\">\n <strong>You are using an outdated browser!</strong> Some features might not look or work like expected. Kattis supports the last two versions of major browsers. Please consider upgrading to a recent version! </div>\n <![endif]-->\n\n \n \n <div class=\"wrap\">\n <div id=\"messages\">\n \n <div class=\"alert alert-dismissible alert-info\">\n <button type=\"button\" class=\"close\" data-dismiss=\"alert\" aria-label=\"Close\">\n <span aria-hidden=\"true\">×</span>\n </button>\n <strong>The page you are trying to access requires you to be logged in.</strong>\n </div>\n </div>\n </div>\n \n \n \n\n <div class=\"wrap\">\n \n\n\n\n\n\n\n\n\n\n \n \n\n <div class=\"page-content boxed clearfix\">\n <section class=\"box clearfix main-content\">\n \n \n\t\n <div class=\"page-headline clearfix\">\n <div style=\"text-align:center\">\n <h1>Log in or sign up for Kattis</h1>\n </div>\n </div>\n\n <br />\n\n <div class=\"login\">\n <div class=\"login-left\">\n <img src=\"/images/kattis/judge.png?7f7dbf=\" alt=\"\" />\n </div>\n\n <div class=\"login-right\">\n\n\t\n <div class=\"login-methods\">\n\n \t\t \n <form action=\"/oauth/Azure\" method=\"GET\" style=\"display:inline-block\">\n <button class=\"Azure\">\n\n <i class=\"fa fa-windows\"></i>\n \n Log in with Azure\n </button>\n </form>\n\n\t\t\t\t\t\t\t\t<br/> \n <form action=\"/oauth/Facebook\" method=\"GET\" style=\"display:inline-block\">\n <button class=\"Facebook\">\n\n <i class=\"fa fa-facebook\"></i>\n \n Log in with Facebook\n </button>\n </form>\n\n\t\t\t\t\t\t\t\t<br/> \n <form action=\"/oauth/Github\" method=\"GET\" style=\"display:inline-block\">\n <button class=\"Github\">\n\n <i class=\"fa fa-github\"></i>\n \n Log in with Github\n </button>\n </form>\n\n\t\t\t\t\t\t\t\t<br/> \n <form action=\"/oauth/Google\" method=\"GET\" style=\"display:inline-block\">\n <button class=\"Google\">\n\n <i class=\"fa fa-google\"></i>\n \n Log in with Google\n </button>\n </form>\n\n\t\t\t\t\t\t\t\t<br/> \n <form action=\"/oauth/LinkedIn\" method=\"GET\" style=\"display:inline-block\">\n <button class=\"LinkedIn\">\n\n <i class=\"fa fa-linkedin\"></i>\n \n Log in with LinkedIn\n </button>\n </form>\n\n\t\t\t\t\t\t\t\t<br/> \n\t\t\n\t\t\n <form action=\"/login/email\" method=\"GET\" style=\"display:inline-block\">\n <button class=\"email\">\n <i class=\"fa fa-envelope\"></i>\n Log in with e-mail </button>\n\n <input type=\"hidden\" name=\"todo\" value=\"redirect\" />\n </form>\n \n </div>\n\n\t<br/>\n\t<br/><a href=\"/login/more?todo=redirect\">More login methods</a>\t\n </div></div>\n\n\n </section>\n </div>\n </div>\n\n\n</div>\n\n\n<div id=\"footer\">\n <div class=\"container\">\n <div class=\"row\">\n <div class=\"footer-info col-md-2 \">\n \n </div>\n <div class=\"footer-powered col-md-8\">\n <h4>\n <a href=\"/rss/new-problems\"><i class=\"fa fa-rss-square\" style=\"color: orange\"></i> RSS feed for new problems</a> |\n Powered by Kattis | <a href=\"https://www.patreon.com/kattis\">Support Kattis on Patreon!</a>\n </h4>\n </div>\n </div>\n </div>\n</div>\n\n\n\n\n<script src=\"//cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js\"></script>\n<script src=\"//cdnjs.cloudflare.com/ajax/libs/moment.js/2.24.0/moment.min.js\"></script>\n<script src=\"//cdnjs.cloudflare.com/ajax/libs/bootstrap-datetimepicker/4.17.47/js/bootstrap-datetimepicker.min.js\"></script>\n<script src=\"//cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.min.js\"></script>\n<script src=\"//cdnjs.cloudflare.com/ajax/libs/select2/3.5.4/select2.min.js\"></script>\n<script src=\"//cdnjs.cloudflare.com/ajax/libs/raphael/2.2.8/raphael.min.js\"></script>\n<script src=\"/js/system.js?203d73=\" type=\"text/javascript\"></script>\n\n\n\n\n</body>\n</html>\n"
There's some disparity in the POST requests, but I can't figure out exactly what. I also think I'm able to login with the first request, but I'm not entirely sure the cookies carry over. Is there a general way to rewrite the Python requests POST in Rust? Specifically I think I need the files part to be included.
287
asked Sep 13 '20 16:09
You are not using it, but with requests you'd use a session object to handle cookie persistence. You already found the equivalent in reqwest; a ClientBuilder has a cookie store method which enables the same functionality. Use the builder configured with this to create both requests, and any cookies on one response then are passed on to the next request (following the normal rules for cookie domains, paths and flags).
Next, the requests.post() method combines fields passed to files and data into a single multipart form request body. This does not post JSON data, don't use the RequestBuilder.json() method here. Just add those fields to the multipart request as a text field, using the Form.text() method.
Your login function is also not sending JSON; a dictionary passed to data is handled as form fields instead.
So this should work:
use std::path::Path;
use tokio::fs::File;
// UA string to pass to ClientBuilder.user_agent
let &'static user_agent = "kattis-cli-submit";
let config = get_config().await?;
let client = reqwest::ClientBuilder::new()
.user_agent(user_agent)
.cookie_store(true)
.build()?;
// Login
// could also use a HashMap
let login_fields = [
("user", config.username.as_str()),
("script", "true"),
("token", config.token.as_str()),
];
let login_response = client
.post(&config.login_url)
.form(&login_fields)
.send()
.await?;
println!("{}", login_response);
// Make a submission
let mut form = reqwest::multipart::Form::new()
.text("submit", "true")
.text("submit_ctr", "2")
.text("language", language)
.text("mainclass", problem)
.text("problem", problem)
.text("script", "true");
// add a single file, and set the part filename to the base name of the file path
let path = Path::new(submission_filename);
let sub_file_contents = std::fs::read(path)?;
let sub_file_part = reqwest::multipart::Part::bytes(sub_file_contents)
.file_name(path.file_name().unwrap().to_string_lossy())
.mime_str("application/octet-stream")?;
form = form.part("sub_file[]", sub_file_part);
let submission_response = client
.post(&config.submit_url)
.multipart(form)
.send()
.await?
.text()
.await?;
println!("Submission response:\n{}", submission_response);
I've made use of the ClientBuilder.user_agent() method, rather than manually build a header map, to set the User-Agent string.
Note that the code posts a single file, and reads the file contents into memory first; the multipart::Part::bytes() method produces a new Part that then is further configured by attaching the filename and the mimetype.
I can heartily recommend that you try out posting to https://httpbin.org/post to see what exactly your code ends up sending, and compare that with the Python version.
I’ve created repl.it demos of the code that use httpbin (with some adjustments to work without a config object, plus the code sets a cookie so we can verify that it is being propagated, uploades more than one file, and sets unique part names for the attached files so httpbin shows them properly):
You can see there that the responses from httpbin are the same.
The Python code reads each file into memory to post it; this is not that efficient and limits the file sizes that can be sent with this code. That's probably fine for this script, but for larger files you want to stream the file data straight from disk to the network socket as you send the form data:
use std::path::Path;
use tokio::fs::File;
use tokio_util::codec::{BytesCodec, FramedRead};
let path = Path::new(submission_filename);
// Create a Stream for the attached file, wrapped in a reqwest::Body
let file = File::open(path).await?;
let reader = FramedRead::new(file, BytesCodec::new());
let sub_file_part = reqwest::multipart::Part::stream(Body::wrap_stream(reader))
.file_name(path.file_name().unwrap().to_string_lossy())
.mime_str("application/octet-stream")?;
form = form.part(part_name, sub_file_part);
You can see this in action at https://repl.it/@mjpieters/so63873082-rust-streams#so63873082/src/main.rs
149
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With
