GitHub/decky-loader
1
0
Fork 0
mirror of https://github.com/SteamDeckHomebrew/decky-loader.git synced 2026-06-13 04:05:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

implement fetch and external resource request apis

Browse Source
This commit is contained in:
AAGaming
2024-05-04 22:39:30 -04:00
parent 2a22f000c1
commit 14ea7b964f
14 changed files with 609 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/defsettings.json
Vendored
+1
View File
@@ -1,6 +1,7 @@
{
"deckip" : "0.0.0.0",
"deckport" : "22",
"deckuser" : "deck",
"deckpass" : "ssap",
"deckkey" : "-i ${env:HOME}/.ssh/id_rsa",
"deckdir" : "/home/deck"
.vscode/tasks.json
Vendored
+4 -4
View File
@@ -41,7 +41,7 @@
"deploy"
],
"detail": "Check for local runs, create a plugins folder",
"command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'python -m ensurepip && python -m pip install --user --upgrade poetry && cd ${config:deckdir}/homebrew/dev/pluginloader/backend && python -m poetry install'",
"command": "ssh ${config:deckuser}@${config:deckip} -p ${config:deckport} ${config:deckkey} 'python -m ensurepip && python -m pip install --user --upgrade poetry && cd ${config:deckdir}/homebrew/dev/pluginloader/backend && python -m poetry install'",
"problemMatcher": []
},
{
@@ -100,7 +100,7 @@
"dependsOn": [
"checkforsettings"
],
"command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'mkdir -p ${config:deckdir}/homebrew/dev/pluginloader && mkdir -p ${config:deckdir}/homebrew/plugins'",
"command": "ssh ${config:deckuser}@${config:deckip} -p ${config:deckport} ${config:deckkey} 'mkdir -p ${config:deckdir}/homebrew/dev/pluginloader && mkdir -p ${config:deckdir}/homebrew/plugins'",
"problemMatcher": []
},
{
@@ -108,7 +108,7 @@
"detail": "Deploy dev PluginLoader to deck",
"type": "shell",
"group": "none",
"command": "rsync -azp --delete --rsh='ssh -p ${config:deckport} ${config:deckkey}' --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='backend/**/__pycache__/' --exclude='.gitignore' . deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader",
"command": "rsync -azp --delete --force --rsh='ssh -p ${config:deckport} ${config:deckkey}' --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='**/__pycache__/' --exclude='.gitignore' . ${config:deckuser}@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader",
"problemMatcher": []
},
// RUN
@@ -120,7 +120,7 @@
"dependsOn": [
"checkforsettings"
],
"command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'export PATH=${config:deckdir}/.local/bin:$PATH; export PLUGIN_PATH=${config:deckdir}/homebrew/plugins; export CHOWN_PLUGIN_PATH=0; export LOG_LEVEL=DEBUG; cd ${config:deckdir}/homebrew/dev/pluginloader/backend; echo '${config:deckpass}' | sudo -SE poetry run sh -c \"cd ${config:deckdir}/homebrew/services; python3 ${config:deckdir}/homebrew/dev/pluginloader/backend/main.py\"'",
"command": "ssh ${config:deckuser}@${config:deckip} -p ${config:deckport} ${config:deckkey} 'export PATH=${config:deckdir}/.local/bin:$PATH; export PLUGIN_PATH=${config:deckdir}/homebrew/plugins; export CHOWN_PLUGIN_PATH=0; export LOG_LEVEL=DEBUG; cd ${config:deckdir}/homebrew/dev/pluginloader/backend; echo '${config:deckpass}' | poetry run sh -c \"cd ${config:deckdir}/homebrew/services; sudo -SE env \"PATH=\\$PATH\" python3 ${config:deckdir}/homebrew/dev/pluginloader/backend/main.py\"'",
"problemMatcher": []
},
{
backend/decky_loader/helpers.py
+13 -8
View File
@@ -1,3 +1,4 @@
from platform import version
import re
import ssl
import uuid
@@ -14,7 +15,7 @@ from aiohttp import ClientSession
from .localplatform import localplatform
from .enums import UserType
from logging import getLogger
from packaging.version import Version
from packaging.version import Version # type: ignore
REMOTE_DEBUGGER_UNIT = "steam-web-debug-portforward.service"
@@ -36,12 +37,13 @@ def get_csrf_token():
@middleware
async def csrf_middleware(request: Request, handler: Handler):
if str(request.method) == "OPTIONS" or \
request.headers.get('Authentication') == csrf_token or \
request.headers.get('X-Decky-Auth') == csrf_token or \
str(request.rel_url) == "/auth/token" or \
str(request.rel_url).startswith("/plugins/load_main/") or \
str(request.rel_url).startswith("/static/") or \
str(request.rel_url).startswith("/steam_resource/") or \
str(request.rel_url).startswith("/frontend/") or \
str(request.rel_url.path) == "/fetch" or \
str(request.rel_url.path) == "/ws" or \
assets_regex.match(str(request.rel_url)) or \
dist_regex.match(str(request.rel_url)) or \
@@ -61,24 +63,27 @@ def mkdir_as_user(path: str):
localplatform.chown(path)
# Fetches the version of loader
# TODO THIS IS ABSOLUTELY TERRIBLE AND NEVER SHOULDVE BEEN MERGED! packaging HAS NO TYPES AND WE COULD LITERALLY JUST USE A REGEX!!!!! REWRITE THIS!!!!!!!!!!!!!
def get_loader_version() -> str:
try:
# Normalize Python-style version to conform to Decky style
v = Version(importlib.metadata.version("decky_loader"))
v = Version(importlib.metadata.version("decky_loader")) # type: ignore
version_str = f'v{v.major}.{v.minor}.{v.micro}'
version_str = f'v{v.major}.{v.minor}.{v.micro}' # type: ignore
if v.pre:
version_str += f'-pre{v.pre[1]}'
if v.pre: # type: ignore
version_str += f'-pre{v.pre[1]}' # type: ignore
if v.post:
version_str += f'-dev{v.post}'
if v.post: # type: ignore
version_str += f'-dev{v.post}' # type: ignore
return version_str
except Exception as e:
logger.warn(f"Failed to execute get_loader_version(): {str(e)}")
return "unknown"
user_agent = f"Decky/{get_loader_version()} (https://decky.xyz)"
# returns the appropriate system python paths
def get_system_pythonpaths() -> list[str]:
try:
backend/decky_loader/plugin/imports/decky.py
+1 -1
View File
@@ -12,7 +12,7 @@ Some basic migration helpers are available: `migrate_any`, `migrate_settings`, `
A logging facility `logger` is available which writes to the recommended location.
"""
__version__ = '0.1.0'
__version__ = '1.0.0'
import os
import subprocess
backend/decky_loader/plugin/imports/decky.pyi
+1 -1
View File
@@ -12,7 +12,7 @@ Some basic migration helpers are available: `migrate_any`, `migrate_settings`, `
A logging facility `logger` is available which writes to the recommended location.
"""
__version__ = '0.1.0'
__version__ = '1.0.0'
import logging
backend/decky_loader/plugin/plugin.py
+1
View File
@@ -47,6 +47,7 @@ class PluginWrapper:
self.emitted_event_callback: EmittedEventCallbackType = emit_callback
# TODO enable this after websocket release
self.legacy_method_warning = False
def __str__(self) -> str:
backend/decky_loader/utilities.py
+91 -3
View File
@@ -1,6 +1,8 @@
from __future__ import annotations
from os import stat_result
import uuid
from urllib.parse import unquote
from json.decoder import JSONDecodeError
from os.path import splitext
import re
from traceback import format_exc
@@ -8,6 +10,7 @@ from stat import FILE_ATTRIBUTE_HIDDEN # type: ignore
from asyncio import StreamReader, StreamWriter, start_server, gather, open_connection
from aiohttp import ClientSession
from aiohttp.web import Request, StreamResponse, Response, json_response, post
from typing import TYPE_CHECKING, Callable, Coroutine, Dict, Any, List, TypedDict
from logging import getLogger
@@ -26,12 +29,17 @@ class FilePickerObj(TypedDict):
filest: stat_result
is_dir: bool
decky_header_regex = re.compile("X-Decky-(.*)")
extra_header_regex = re.compile("X-Decky-Header-(.*)")
excluded_default_headers = ["Host", "Origin", "Sec-Fetch-Site", "Sec-Fetch-Mode", "Sec-Fetch-Dest"]
class Utilities:
def __init__(self, context: PluginManager) -> None:
self.context = context
self.legacy_util_methods: Dict[str, Callable[..., Coroutine[Any, Any, Any]]] = {
"ping": self.ping,
"http_request": self.http_request,
"http_request": self.http_request_legacy,
"install_plugin": self.install_plugin,
"install_plugins": self.install_plugins,
"cancel_plugin_install": self.cancel_plugin_install,
@@ -76,9 +84,33 @@ class Utilities:
context.ws.add_route("utilities/enable_rdt", self.enable_rdt)
context.ws.add_route("utilities/get_tab_id", self.get_tab_id)
context.ws.add_route("utilities/get_user_info", self.get_user_info)
context.ws.add_route("utilities/http_request", self.http_request)
context.ws.add_route("utilities/http_request", self.http_request_legacy)
context.ws.add_route("utilities/_call_legacy_utility", self._call_legacy_utility)
context.web_app.add_routes([
post("/methods/{method_name}", self._handle_legacy_server_method_call)
])
for method in ('GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD'):
context.web_app.router.add_route(method, "/fetch", self.http_request)
async def _handle_legacy_server_method_call(self, request: Request) -> Response:
method_name = request.match_info["method_name"]
try:
args = await request.json()
except JSONDecodeError:
args = {}
res = {}
try:
r = await self.legacy_util_methods[method_name](**args)
res["result"] = r
res["success"] = True
except Exception as e:
res["result"] = str(e)
res["success"] = False
return json_response(res)
async def _call_legacy_utility(self, method_name: str, kwargs: Dict[Any, Any]) -> Any:
self.logger.debug(f"Calling utility {method_name} with legacy kwargs");
res: Dict[Any, Any] = {}
@@ -114,7 +146,63 @@ class Utilities:
async def uninstall_plugin(self, name: str):
return await self.context.plugin_browser.uninstall_plugin(name)
async def http_request(self, method: str, url: str, extra_opts: Any = {}):
# Loosely based on https://gist.github.com/mosquito/4dbfacd51e751827cda7ec9761273e95#file-proxy-py
async def http_request(self, req: Request) -> StreamResponse:
if req.headers.get('X-Decky-Auth', '') != helpers.get_csrf_token() and req.query.get('auth', '') != helpers.get_csrf_token():
return Response(text='Forbidden', status=403)
url = req.headers["X-Decky-Fetch-URL"] if "X-Decky-Fetch-URL" in req.headers else unquote(req.query.get('fetch_url', ''))
self.logger.info(f"Preparing {req.method} request to {url}")
headers = dict(req.headers)
headers["User-Agent"] = helpers.user_agent
for excluded_header in excluded_default_headers:
self.logger.debug(f"Excluding default header {excluded_header}")
if excluded_header in headers:
del headers[excluded_header]
if "X-Decky-Fetch-Excluded-Headers" in req.headers:
for excluded_header in req.headers["X-Decky-Fetch-Excluded-Headers"].split(", "):
self.logger.debug(f"Excluding header {excluded_header}")
if excluded_header in headers:
del headers[excluded_header]
for header in req.headers:
match = extra_header_regex.search(header)
if match:
header_name = match.group(1)
header_value = req.headers[header]
self.logger.debug(f"Adding extra header {header_name}: {header_value}")
headers[header_name] = header_value
for header in list(headers.keys()):
match = decky_header_regex.search(header)
if match:
self.logger.debug(f"Removing decky header {header} from request")
del headers[header]
self.logger.debug(f"Final request headers: {headers}")
body = await req.read() # TODO can this also be streamed?
async with ClientSession() as web:
async with web.request(req.method, url, headers=headers, data=body, ssl=helpers.get_ssl_context()) as web_res:
res = StreamResponse(headers=web_res.headers, status=web_res.status)
if web_res.headers.get('Transfer-Encoding', '').lower() == 'chunked':
res.enable_chunked_encoding()
await res.prepare(req)
self.logger.debug(f"Starting stream for {url}")
async for data in web_res.content.iter_any():
await res.write(data)
if data:
await res.drain()
self.logger.debug(f"Finished stream for {url}")
return res
async def http_request_legacy(self, method: str, url: str, extra_opts: Any = {}):
async with ClientSession() as web:
res = await web.request(method, url, ssl=helpers.get_ssl_context(), **extra_opts)
text = await res.text()
backend/decky_loader/wsrouter.py
+1 -1
View File
@@ -117,7 +117,7 @@ class WSRouter:
create_task(self._call_route(data["route"], data["args"], data["id"]))
else:
error = {"error":f'Route {data["route"]} does not exist.', "name": "RouteNotFoundError", "traceback": None}
create_task(self.write({"type": MessageType.ERROR.value, "id": data["id"], "message": error}))
create_task(self.write({"type": MessageType.ERROR.value, "id": data["id"], "error": error}))
case _:
self.logger.error("Unknown message type", data)
finally:
backend/poetry.lock
Generated
+439 -487
View File
File diff suppressed because it is too large Load Diff
backend/pyinstaller.spec
+1 -1
View File
@@ -9,7 +9,7 @@ a = Analysis(
('locales', 'locales'),
('static', 'static'),
] + copy_metadata('decky_loader'),
hiddenimports=['logging.handlers', 'sqlite3', 'decky_plugin'],
hiddenimports=['logging.handlers', 'sqlite3', 'decky_plugin' 'decky'],
)
pyz = PYZ(a.pure, a.zipped_data)
backend/pyproject.toml
+2 -1
View File
@@ -13,12 +13,13 @@ include = ["decky_loader/static/*"]
[tool.poetry.dependencies]
python = ">=3.10,<3.13"
aiohttp = "^3.8.5"
aiohttp = "^3.9.5"
aiohttp-jinja2 = "^1.5.1"
aiohttp-cors = "^0.7.0"
watchdog = "^2.1.7"
certifi = "*"
packaging = "^23.2"
multidict = "^6.0.5"
[tool.poetry.group.dev.dependencies]
pyinstaller = "^5.13.0"
frontend/src/plugin-loader.tsx
+49 -13
View File
@@ -42,7 +42,7 @@ const FilePicker = lazy(() => import('./components/modals/filepicker'));
declare global {
interface Window {
__DECKY_SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED_deckyPluginBackendAPIInit?: {
__DECKY_SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED_deckyLoaderAPIInit?: {
connect: (version: number, key: string) => any; // Returns the backend API used above, no real point adding types to this.
};
}
@@ -51,6 +51,10 @@ declare global {
/** Map of event names to event listeners */
type listenerMap = Map<string, Set<(...args: any) => any>>;
interface DeckyRequestInit extends RequestInit {
excludedHeaders: string[];
}
const callPluginMethod = DeckyBackend.callable<[pluginName: string, method: string, ...args: any], any>(
'loader/call_plugin_method',
);
@@ -357,7 +361,7 @@ class PluginLoader extends Logger {
let res = await fetch(`http://127.0.0.1:1337/plugins/${name}/frontend_bundle`, {
credentials: 'include',
headers: {
Authentication: deckyAuthToken,
'X-Decky-Auth': deckyAuthToken,
},
});
if (res.ok) {
@@ -484,12 +488,31 @@ class PluginLoader extends Logger {
});
}
/* TODO replace with the following flow (or similar) so we can reuse the JS Fetch API
frontend --request URL only--> backend (ws method)
backend --new temporary backend URL--> frontend (ws response)
frontend <--> backend <--> target URL (over http!)
*/
async fetchNoCors(url: string, request: any = {}) {
// Useful for audio/video streams
getExternalResourceURL(url: string) {
return `http://127.0.0.1:1337/fetch?auth=${deckyAuthToken}&fetch_url=${encodeURIComponent(url)}`;
}
// Same syntax as fetch but only supports the url-based syntax and an object for headers since it's the most common usage pattern
fetch(input: string, init?: DeckyRequestInit | undefined): Promise<Response> {
const headers: { [name: string]: string } = {
...(init?.headers as { [name: string]: string }),
'X-Decky-Auth': deckyAuthToken,
'X-Decky-Fetch-URL': input,
};
if (init?.excludedHeaders) {
headers['X-Decky-Fetch-Excluded-Headers'] = init.excludedHeaders.join(', ');
}
return fetch('http://127.0.0.1:1337/fetch', {
...init,
credentials: 'include',
headers,
});
}
async legacyFetchNoCors(url: string, request: any = {}) {
let method: string;
const req = { headers: {}, ...request, data: request.body };
req?.body && delete req.body;
@@ -513,10 +536,10 @@ class PluginLoader extends Logger {
initPluginBackendAPI() {
// Things will break *very* badly if plugin code touches this outside of @decky/backend, so lets make that clear.
window.__DECKY_SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED_deckyPluginBackendAPIInit = {
window.__DECKY_SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED_deckyLoaderAPIInit = {
connect: (version: number, pluginName: string) => {
if (version <= 0) {
throw new Error(`Plugin ${pluginName} requested invalid backend api version ${version}.`);
if (version < 1 || version > 1) {
throw new Error(`Plugin ${pluginName} requested unsupported backend api version ${version}.`);
}
const eventListeners: listenerMap = new Map();
@@ -543,9 +566,22 @@ class PluginLoader extends Logger {
set?.delete(listener);
}
},
openFilePicker: this.openFilePicker.bind(this),
executeInTab: DeckyBackend.callable<
[tab: String, runAsync: Boolean, code: string],
{ success: boolean; result: any }
>('utilities/execute_in_tab'),
fetch: this.fetch.bind(this),
getExternalResourceURL: this.getExternalResourceURL.bind(this),
injectCssIntoTab: DeckyBackend.callable<[tab: string, style: string], string>(
'utilities/inject_css_into_tab',
),
removeCssFromTab: DeckyBackend.callable<[tab: string, cssId: string]>('utilities/remove_css_from_tab'),
routerHook: this.routerHook,
toaster: this.toaster,
};
this.debug(`${pluginName} connected to backend API.`);
this.debug(`${pluginName} connected to loader API.`);
return backendAPI;
},
};
@@ -591,7 +627,7 @@ class PluginLoader extends Logger {
args,
);
},
fetchNoCors: this.fetchNoCors,
fetchNoCors: this.legacyFetchNoCors,
executeInTab: DeckyBackend.callable<
[tab: String, runAsync: Boolean, code: string],
{ success: boolean; result: any }
frontend/src/start.tsx
+1 -1
View File
@@ -32,7 +32,7 @@ declare global {
backend: {
loadPath: 'http://127.0.0.1:1337/locales/{{lng}}.json',
customHeaders: {
Authentication: deckyAuthToken,
'X-Decky-Auth': deckyAuthToken,
},
requestOptions: {
credentials: 'include',
frontend/src/wsrouter.ts
+4 -4
View File
@@ -30,7 +30,7 @@ interface ReplyMessage {
interface ErrorMessage {
type: MessageType.ERROR;
error: { name: string; message: string; traceback: string | null };
error: { name: string; error: string; traceback: string | null };
id: number;
}
@@ -40,8 +40,8 @@ interface ErrorMessage {
export class PyError extends Error {
pythonTraceback: string | null;
constructor(name: string, message: string, traceback: string | null) {
super(message);
constructor(name: string, error: string, traceback: string | null) {
super(error);
this.name = `Python ${name}`;
if (traceback) {
// traceback will always start with `Traceback (most recent call last):`
@@ -142,7 +142,7 @@ export class WSRouter extends Logger {
case MessageType.ERROR:
if (this.runningCalls.has(data.id)) {
let err = new PyError(data.error.name, data.error.message, data.error.traceback);
let err = new PyError(data.error.name, data.error.error, data.error.traceback);
this.runningCalls.get(data.id)!.reject(err);
this.runningCalls.delete(data.id);
this.debug(`Rejected PY call ${data.id} with error`, data.error);