-
Notifications
You must be signed in to change notification settings - Fork 111
[k2] add support http multipart content-type #1545
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
astrophysik
wants to merge
38
commits into
master
Choose a base branch
from
vsadokhov/k2-http-multipart
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
38 commits
Select commit
Hold shift + click to select a range
8460a39
[k2] add support multipart/form-data to HTTP server
nekishdev c6800d1
[k2] refactor multipart/form-data via ranges
nekishdev 95f8761
wip
astrophysik 3cf7a2f
wip
astrophysik 858b5da
wip
astrophysik 3498cfe
wip
astrophysik a75db65
wip
astrophysik db552d2
wip
astrophysik 1feac9b
add tests
astrophysik dc53e7a
small fix
astrophysik 70155f6
small fix
astrophysik 45028bd
small fix
astrophysik 49c97cc
small fix
astrophysik 1eeb6d5
small fix
astrophysik 6ff99ad
small fix
astrophysik 7402059
small fix
astrophysik d55d677
apply review
astrophysik 2a5a562
add test assert
astrophysik 8fb5338
fix format
astrophysik a3e9d1c
fix format
astrophysik 8b2afd3
apply review
astrophysik 87b958b
fix format
astrophysik 7b4eed0
remove extra const
astrophysik e6fd803
add include
astrophysik 9dc1aec
apply review
astrophysik 383ff2d
fix format
astrophysik 31734c5
small fixes
astrophysik 197ac4a
fix format
astrophysik 6233a90
small fix
astrophysik e9bca64
add inline for headers constants
astrophysik 5465ad7
small fixes
astrophysik 409672d
add multipart_temporary_files clear
astrophysik 246305f
remove saving extra temp files
astrophysik c1e793d
small fixes
astrophysik 23a2b9a
some code improvements
astrophysik 6355069
apply format
astrophysik 5787687
remove some const
astrophysik 6285b2c
add more tests
astrophysik File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
176 changes: 176 additions & 0 deletions
176
runtime-light/server/http/multipart/details/parts-parsing.h
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,176 @@ | ||
| // Compiler for PHP (aka KPHP) | ||
| // Copyright (c) 2026 LLC «V Kontakte» | ||
| // Distributed under the GPL v3 License, see LICENSE.notice.txt | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <algorithm> | ||
| #include <cstddef> | ||
| #include <locale> | ||
| #include <optional> | ||
| #include <ranges> | ||
| #include <string> | ||
| #include <string_view> | ||
| #include <utility> | ||
|
|
||
| #include "common/algorithms/string-algorithms.h" | ||
| #include "runtime-light/server/http/http-server-state.h" | ||
|
|
||
| namespace kphp::http::multipart::details { | ||
|
|
||
| inline constexpr std::string_view HEADER_CONTENT_DISPOSITION_FORM_DATA = "form-data;"; | ||
|
|
||
| inline std::string_view trim_crlf(std::string_view sv) noexcept { | ||
| if (sv.starts_with('\r')) { | ||
| sv.remove_prefix(1); | ||
| } | ||
| if (sv.starts_with('\n')) { | ||
| sv.remove_prefix(1); | ||
| } | ||
|
|
||
| if (sv.ends_with('\n')) { | ||
| sv.remove_suffix(1); | ||
| } | ||
| if (sv.ends_with('\r')) { | ||
| sv.remove_suffix(1); | ||
| } | ||
| return sv; | ||
| } | ||
|
|
||
| struct part_header { | ||
| std::string_view name; | ||
| std::string_view value; | ||
|
|
||
| static std::optional<part_header> parse(std::string_view header) noexcept { | ||
| auto [name_view, value_view]{vk::split_string_view(header, ':')}; | ||
| name_view = vk::trim(name_view); | ||
| value_view = vk::trim(value_view); | ||
| if (name_view.empty() || value_view.empty()) { | ||
| return std::nullopt; | ||
| } | ||
| return part_header{name_view, value_view}; | ||
| } | ||
|
|
||
| bool name_is(std::string_view header_name) const noexcept { | ||
| const auto lower_name{name | std::views::transform([](auto c) noexcept { return std::tolower(c, std::locale::classic()); })}; | ||
| const auto lower_header_name{header_name | std::views::transform([](auto c) noexcept { return std::tolower(c, std::locale::classic()); })}; | ||
| return std::ranges::equal(lower_name, lower_header_name); | ||
| } | ||
|
|
||
| private: | ||
| part_header(std::string_view name, std::string_view value) noexcept | ||
| : name(name), | ||
| value(value) {} | ||
| }; | ||
|
|
||
| inline auto parse_headers(std::string_view sv) noexcept { | ||
| static constexpr std::string_view DELIM = "\r\n"; | ||
| return std::views::split(sv, DELIM) | std::views::transform([](auto raw_header) noexcept { return part_header::parse(std::string_view(raw_header)); }) | | ||
| std::views::take_while([](auto header_opt) noexcept { return header_opt.has_value(); }) | | ||
| std::views::transform([](auto header_opt) noexcept { return *header_opt; }); | ||
| } | ||
|
|
||
| struct part_attribute { | ||
| std::string_view name; | ||
| std::string_view value; | ||
|
|
||
| static std::optional<part_attribute> parse(std::string_view attribute) noexcept { | ||
| auto [name_view, value_view]{vk::split_string_view(vk::trim(attribute), '=')}; | ||
| name_view = vk::trim(name_view); | ||
| value_view = vk::trim(value_view); | ||
| if (name_view.empty() || value_view.empty()) { | ||
| return std::nullopt; | ||
| } | ||
|
|
||
| if (value_view.starts_with('"') && value_view.ends_with('"') && value_view.size() > 1) { | ||
| value_view.remove_suffix(1); | ||
| value_view.remove_prefix(1); | ||
| } | ||
| return part_attribute{name_view, value_view}; | ||
| } | ||
|
|
||
| private: | ||
| part_attribute(std::string_view name, std::string_view value) noexcept | ||
| : name(name), | ||
| value(value) {} | ||
| }; | ||
|
|
||
| inline auto parse_attrs(std::string_view header_value) noexcept { | ||
| static constexpr std::string_view DELIM = ";"; | ||
| return std::views::split(header_value, DELIM) | std::views::transform([](auto part) noexcept { return part_attribute::parse(std::string_view(part)); }) | | ||
| std::views::take_while([](auto attribute_opt) noexcept { return attribute_opt.has_value(); }) | | ||
| std::views::transform([](auto attribute_opt) noexcept { return *attribute_opt; }); | ||
| } | ||
|
|
||
| struct part { | ||
| std::string_view name_attribute; | ||
| std::optional<std::string_view> filename_attribute; | ||
| std::optional<std::string_view> content_type; | ||
| std::string_view body; | ||
|
|
||
| static std::optional<part> parse(std::string_view part_view) noexcept { | ||
| static constexpr std::string_view PART_BODY_DELIM = "\r\n\r\n"; | ||
|
|
||
| const size_t part_body_start{part_view.find(PART_BODY_DELIM)}; | ||
| if (part_body_start == std::string_view::npos) { | ||
| return std::nullopt; | ||
| } | ||
|
|
||
| const std::string_view part_headers{part_view.substr(0, part_body_start)}; | ||
| const std::string_view part_body{part_view.substr(part_body_start + PART_BODY_DELIM.size())}; | ||
|
|
||
| std::optional<std::string_view> content_type{std::nullopt}; | ||
| std::optional<std::string_view> filename_attribute{std::nullopt}; | ||
| std::optional<std::string_view> name_attribute{std::nullopt}; | ||
|
|
||
| for (const auto& header : parse_headers(part_headers)) { | ||
| if (header.name_is(kphp::http::headers::CONTENT_DISPOSITION)) { | ||
| if (!header.value.starts_with(HEADER_CONTENT_DISPOSITION_FORM_DATA)) { | ||
| return std::nullopt; | ||
| } | ||
|
|
||
| // skip first Content-Disposition: form-data; | ||
| const size_t pos{header.value.find(';')}; | ||
| if (pos == std::string::npos) { | ||
| return std::nullopt; | ||
| } | ||
|
|
||
| const std::string_view attributes{trim_crlf(header.value).substr(pos + 1)}; | ||
| for (const auto& attribute : parse_attrs(attributes)) { | ||
| if (attribute.name == "name") { | ||
| name_attribute = attribute.value; | ||
| } else if (attribute.name == "filename") { | ||
| filename_attribute = attribute.value; | ||
| } else { | ||
| // ignore unknown attribute | ||
| } | ||
| } | ||
| } else if (header.name_is(kphp::http::headers::CONTENT_TYPE)) { | ||
| content_type = trim_crlf(header.value); | ||
| } else { | ||
| // ignore unused header | ||
| } | ||
| } | ||
| if (!name_attribute.has_value() || name_attribute->empty()) { | ||
| return std::nullopt; | ||
| } | ||
| return part(*name_attribute, filename_attribute, content_type, part_body); | ||
| } | ||
|
|
||
| private: | ||
| part(std::string_view name_attribute, std::optional<std::string_view> filename_attribute, std::optional<std::string_view> content_type, | ||
| std::string_view body) noexcept | ||
| : name_attribute(name_attribute), | ||
| filename_attribute(filename_attribute), | ||
| content_type(content_type), | ||
| body(body) {} | ||
| }; | ||
|
|
||
| template<typename Delim> | ||
| auto parse_multipart_parts(std::string_view body, Delim&& delim) noexcept { | ||
| return std::views::split(body, std::forward<Delim>(delim)) | std::views::filter([](auto raw_part) noexcept { return !std::string_view(raw_part).empty(); }) | | ||
| std::views::transform([](auto raw_part) noexcept -> std::optional<part> { return part::parse(trim_crlf(std::string_view(raw_part))); }) | | ||
| std::views::take_while([](auto part_opt) noexcept { return part_opt.has_value(); }) | std::views::transform([](auto part_opt) { return *part_opt; }); | ||
| } | ||
|
|
||
| } // namespace kphp::http::multipart::details |
159 changes: 159 additions & 0 deletions
159
runtime-light/server/http/multipart/details/parts-processing.cpp
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,159 @@ | ||
| // Compiler for PHP (aka KPHP) | ||
| // Copyright (c) 2026 LLC «V Kontakte» | ||
| // Distributed under the GPL v3 License, see LICENSE.notice.txt | ||
|
|
||
| #include "runtime-light/server/http/multipart/details/parts-processing.h" | ||
|
|
||
| #include <algorithm> | ||
| #include <cstddef> | ||
| #include <cstdint> | ||
| #include <cstdio> | ||
| #include <expected> | ||
| #include <optional> | ||
| #include <ranges> | ||
| #include <span> | ||
| #include <string_view> | ||
| #include <unistd.h> | ||
| #include <utility> | ||
|
|
||
| #include "runtime-common/core/allocator/script-allocator.h" | ||
| #include "runtime-common/core/runtime-core.h" | ||
| #include "runtime-common/core/std/containers.h" | ||
| #include "runtime-common/stdlib/server/url-functions.h" | ||
| #include "runtime-light/k2-platform/k2-api.h" | ||
| #include "runtime-light/server/http/http-server-state.h" | ||
| #include "runtime-light/server/http/multipart/details/parts-parsing.h" | ||
| #include "runtime-light/state/component-state.h" | ||
| #include "runtime-light/stdlib/diagnostics/logs.h" | ||
| #include "runtime-light/stdlib/file/resource.h" | ||
| #include "runtime-light/stdlib/math/random-functions.h" | ||
|
|
||
| namespace { | ||
|
|
||
| constexpr std::string_view CONTENT_TYPE_APP_FORM_URLENCODED = "application/x-www-form-urlencoded"; | ||
|
|
||
| constexpr std::string_view DEFAULT_CONTENT_TYPE = "text/plain"; | ||
|
|
||
| constexpr int32_t UPLOAD_ERR_OK = 0; | ||
| constexpr int32_t UPLOAD_ERR_PARTIAL = 3; | ||
| constexpr int32_t UPLOAD_ERR_NO_FILE = 4; | ||
| constexpr int32_t UPLOAD_ERR_CANT_WRITE = 7; | ||
|
|
||
| // Not implemented : | ||
| // constexpr int32_t UPLOAD_ERR_INI_SIZE = 1; // unused in kphp | ||
| // constexpr int32_t UPLOAD_ERR_FORM_SIZE = 2; // todo support header max-file-size | ||
| // constexpr int32_t UPLOAD_ERR_NO_TMP_DIR = 6; // todo support check tmp dir | ||
| // constexpr int32_t UPLOAD_ERR_EXTENSION = 8; // unused in kphp | ||
|
|
||
| inline constexpr std::string_view LETTERS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; | ||
| inline constexpr int64_t GENERATE_ATTEMPTS = 4; | ||
| inline constexpr int64_t SYMBOLS_COUNT = 6; | ||
|
|
||
| char random_letter() noexcept { | ||
| int64_t pos{f$mt_rand(0, LETTERS.size() - 1)}; | ||
| return LETTERS[pos]; | ||
| } | ||
|
|
||
| std::optional<kphp::stl::string<kphp::memory::script_allocator>> generate_temporary_name() noexcept { | ||
| // todo rework with k2::tempnam or mkstemp | ||
| auto tmp_dir_env{ComponentState::get().env.get_value(string{"TMPDIR"})}; | ||
| std::string_view tmp_path{tmp_dir_env.is_string() ? std::string_view{tmp_dir_env.as_string().c_str(), tmp_dir_env.as_string().size()} : P_tmpdir}; | ||
| for (int64_t attempt{}; attempt < GENERATE_ATTEMPTS; ++attempt) { | ||
| kphp::stl::string<kphp::memory::script_allocator> tmp_name{tmp_path.data(), tmp_path.size()}; | ||
| tmp_name.push_back('/'); | ||
| for (auto _ : std::views::iota(0, SYMBOLS_COUNT)) { | ||
| tmp_name.push_back(random_letter()); | ||
| } | ||
| if (!k2::access(tmp_name, F_OK).has_value()) { | ||
| return tmp_name; | ||
| } | ||
| } | ||
| return std::nullopt; | ||
| } | ||
|
|
||
| std::expected<size_t, int32_t> write_temporary_file(std::string_view tmp_name, std::span<const std::byte> content) noexcept { | ||
apolyakov marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| auto file_res{kphp::fs::file::open(tmp_name, "w")}; | ||
| if (!file_res.has_value()) { | ||
| return std::unexpected{UPLOAD_ERR_NO_FILE}; | ||
| } | ||
|
|
||
| auto written_res{(*file_res).write(content)}; | ||
| if (!written_res.has_value()) { | ||
| std::ignore = k2::unlink(tmp_name); | ||
| return std::unexpected{UPLOAD_ERR_CANT_WRITE}; | ||
| } | ||
|
|
||
| size_t file_size{*written_res}; | ||
| if (file_size < content.size()) { | ||
| std::ignore = k2::unlink(tmp_name); | ||
| return std::unexpected{UPLOAD_ERR_PARTIAL}; | ||
| } | ||
| return file_size; | ||
| } | ||
|
|
||
| array<mixed> build_file_array(std::string_view filename, std::string_view tmp_name, std::string_view content_type, | ||
| std::expected<size_t, int32_t> write_res) noexcept { | ||
| array<mixed> file{}; | ||
| if (!write_res.has_value()) { | ||
| file.set_value(string{"size"}, 0); | ||
| file.set_value(string{"tmp_name"}, string{}); | ||
| file.set_value(string{"error"}, write_res.error()); | ||
| } else { | ||
| file.set_value(string{"name"}, string{filename.data(), static_cast<string::size_type>(filename.size())}); | ||
| file.set_value(string{"type"}, string{content_type.data(), static_cast<string::size_type>(content_type.size())}); | ||
| file.set_value(string{"size"}, static_cast<int64_t>(*write_res)); | ||
| file.set_value(string{"tmp_name"}, string{tmp_name.data(), static_cast<string::size_type>(tmp_name.size())}); | ||
| file.set_value(string{"error"}, UPLOAD_ERR_OK); | ||
| } | ||
| return file; | ||
| } | ||
|
|
||
| void add_file_to_array(array<mixed>& files, std::string_view name_attribute, array<mixed> file) noexcept { | ||
| if (name_attribute.ends_with("[]")) { | ||
| string name{name_attribute.data(), static_cast<string::size_type>(name_attribute.size() - 2)}; | ||
| mixed file_array{files.get_value(name)}; | ||
| for (auto& attribute_it : file) { | ||
| string attribute{attribute_it.get_key().to_string()}; | ||
| mixed file_array_value{file_array.get_value(attribute)}; | ||
| file_array_value.push_back(attribute_it.get_value().to_string()); | ||
| file_array.set_value(attribute, file_array_value); | ||
| } | ||
| files.set_value(name, file_array); | ||
| } else { | ||
| string name{name_attribute.data(), static_cast<string::size_type>(name_attribute.size())}; | ||
| files.set_value(name, file); | ||
| } | ||
| } | ||
|
|
||
| } // namespace | ||
|
|
||
| namespace kphp::http::multipart::details { | ||
|
|
||
| void process_post_multipart(const kphp::http::multipart::details::part& part, array<mixed>& post) noexcept { | ||
| string name{part.name_attribute.data(), static_cast<string::size_type>(part.name_attribute.size())}; | ||
| string body{part.body.data(), static_cast<string::size_type>(part.body.size())}; | ||
| if (part.content_type.has_value() && !std::ranges::search(*part.content_type, CONTENT_TYPE_APP_FORM_URLENCODED).empty()) { | ||
| auto post_value{post.get_value(name)}; | ||
| f$parse_str(body, post_value); | ||
| post.set_value(name, std::move(post_value)); | ||
| } else { | ||
| post.set_value(name, body); | ||
| } | ||
| } | ||
|
|
||
| void process_file_multipart(const kphp::http::multipart::details::part& part, array<mixed>& files) noexcept { | ||
| kphp::log::assertion(part.filename_attribute.has_value()); | ||
| auto tmp_name{*(generate_temporary_name().or_else([]() { // NOLINT | ||
| kphp::log::error("cannot generate unique name for multipart temporary file"); // no return | ||
| return std::optional<kphp::stl::string<kphp::memory::script_allocator>>{}; | ||
| }))}; | ||
| auto body_bytes_span{std::as_bytes(std::span<const char>(part.body.data(), part.body.size()))}; | ||
| auto write_res{write_temporary_file(tmp_name, body_bytes_span)}; | ||
| if (write_res.has_value()) { | ||
| HttpServerInstanceState::get().multipart_temporary_files.insert(tmp_name); | ||
| } | ||
| auto content_type{part.content_type.value_or(DEFAULT_CONTENT_TYPE)}; | ||
| auto file{build_file_array(*part.filename_attribute, tmp_name, content_type, write_res)}; | ||
| add_file_to_array(files, part.name_attribute, std::move(file)); | ||
| } | ||
| } // namespace kphp::http::multipart::details | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.