update wine

And force copy artifacts after.

Makefile

@@ -145,19 +145,24 @@ install-internal: downloads
 
 install: install-internal
 	mkdir -p $(STEAM_DIR)/compatibilitytools.d/
+	rm -rf $(STEAM_DIR)/compatibilitytools.d/$(_build_name)/files/ #remove proton's internal files, but preserve user_settings etc from top-level
 	cp -Rf --no-dereference --preserve=mode,links vagrant_share/compatibilitytools.d/$(_build_name) $(STEAM_DIR)/compatibilitytools.d/
 	echo "Proton installed to your local Steam installation"
 
 redist: | vagrant_share/$(DEPLOY_DIR)
 redist: private SHELL := $(VAGRANT_SHELL)
 redist: downloads
-	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) $(CCACHE_FLAG) redist && cp $(BUILD_DIR)/redist/* /vagrant/$(DEPLOY_DIR) && \
+	rm -rf /vagrant/$(DEPLOY_DIR)/* && \
+	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) $(CCACHE_FLAG) redist && \
+	cp -Rf $(BUILD_DIR)/redist/* /vagrant/$(DEPLOY_DIR) && \
 	echo "Proton build available at vagrant_share/$(DEPLOY_DIR)"
 
 deploy: | vagrant_share/$(DEPLOY_DIR)-deploy
 deploy: private SHELL := $(VAGRANT_SHELL)
 deploy: downloads
-	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) $(CCACHE_FLAG) deploy && cp $(BUILD_DIR)/deploy/* /vagrant/$(DEPLOY_DIR)-deploy && \
+	rm -rf /vagrant/$(DEPLOY_DIR)-deploy/* && \
+	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) $(CCACHE_FLAG) deploy && \
+	cp -Rf $(BUILD_DIR)/deploy/* /vagrant/$(DEPLOY_DIR)-deploy && \
 	echo "Proton deployed to vagrant_share/$(DEPLOY_DIR)-deploy"
 
 module: | vagrant_share/$(module)/lib/wine/
@@ -178,8 +183,8 @@ dxvk: | vagrant_share/dxvk/lib64/wine/dxvk
 dxvk: private SHELL := $(VAGRANT_SHELL)
 dxvk: downloads
 	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) $(CCACHE_FLAG) dxvk && \
-	cp -f $(BUILD_DIR)/dist/dist/lib/wine/dxvk/*.dll /vagrant/dxvk/lib/wine/dxvk/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/dxvk/*.dll /vagrant/dxvk/lib64/wine/dxvk/
+	cp -f $(BUILD_DIR)/dist/files/lib/wine/dxvk/*.dll /vagrant/dxvk/lib/wine/dxvk/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/dxvk/*.dll /vagrant/dxvk/lib64/wine/dxvk/
 
 dxvk-nvapi: | vagrant_share/dxvk-nvapi/lib/wine/nvapi
 dxvk-nvapi: | vagrant_share/dxvk-nvapi/lib64/wine/nvapi
@@ -194,31 +199,31 @@ vkd3d-proton: | vagrant_share/vkd3d-proton/lib64/wine/vkd3d-proton
 vkd3d-proton: private SHELL := $(VAGRANT_SHELL)
 vkd3d-proton: downloads
 	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) $(CCACHE_FLAG) vkd3d-proton && \
-	cp -f $(BUILD_DIR)/dist/dist/lib/wine/vkd3d-proton/*.dll /vagrant/vkd3d-proton/lib/wine/vkd3d-proton/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/vkd3d-proton/*.dll /vagrant/vkd3d-proton/lib64/wine/vkd3d-proton/
+	cp -f $(BUILD_DIR)/dist/files/lib/wine/vkd3d-proton/*.dll /vagrant/vkd3d-proton/lib/wine/vkd3d-proton/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/vkd3d-proton/*.dll /vagrant/vkd3d-proton/lib64/wine/vkd3d-proton/
 
 lsteamclient: | vagrant_share/lsteamclient/lib/wine
 lsteamclient: | vagrant_share/lsteamclient/lib64/wine
 lsteamclient: private SHELL := $(VAGRANT_SHELL)
 lsteamclient: downloads
 	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) $(CCACHE_FLAG) lsteamclient && \
-	cp -f $(BUILD_DIR)/dist/dist/lib/wine/lsteamclient.dll.so /vagrant/lsteamclient/lib/wine && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/lsteamclient.dll.so /vagrant/lsteamclient/lib64/wine
+	cp -f $(BUILD_DIR)/dist/files/lib/wine/lsteamclient.dll.so /vagrant/lsteamclient/lib/wine && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/lsteamclient.dll.so /vagrant/lsteamclient/lib64/wine
 
 vrclient: | vagrant_share/vrclient/lib/wine
 vrclient: | vagrant_share/vrclient/lib64/wine
 vrclient: private SHELL := $(VAGRANT_SHELL)
 vrclient: downloads
 	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) $(CCACHE_FLAG) vrclient && \
-	cp -f $(BUILD_DIR)/dist/dist/lib/wine/vrclient.dll.so /vagrant/vrclient/lib/wine && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/vrclient_x64.dll.so /vagrant/vrclient/lib64/wine
+	cp -f $(BUILD_DIR)/dist/files/lib/wine/vrclient.dll.so /vagrant/vrclient/lib/wine && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/vrclient_x64.dll.so /vagrant/vrclient/lib64/wine
 
 wineopenxr: | vagrant_share/wineopenxr/lib/wine
 wineopenxr: | vagrant_share/wineopenxr/lib64/wine
 wineopenxr: private SHELL := $(VAGRANT_SHELL)
 wineopenxr: downloads
 	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) $(CCACHE_FLAG) wineopenxr && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/wineopenxr.dll.so /vagrant/wineopenxr/lib64/wine
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/wineopenxr.dll.so /vagrant/wineopenxr/lib64/wine
 
 vagrant_share/%:
 	mkdir -p $@

build/makefile_base.mak

@@ -143,7 +143,7 @@ endif
 ##
 
 DST_BASE := $(OBJ)/dist
-DST_DIR := $(DST_BASE)/dist
+DST_DIR := $(DST_BASE)/files
 DST_LIBDIR32 := $(DST_DIR)/lib
 DST_LIBDIR64 := $(DST_DIR)/lib64
 DEPLOY_DIR := ./deploy
@@ -185,6 +185,7 @@ CARGO_BUILD_ARG := --release
 COMPAT_MANIFEST_TEMPLATE := $(SRCDIR)/compatibilitytool.vdf.template
 LICENSE := $(SRCDIR)/dist.LICENSE
 OFL_LICENSE := $(SRCDIR)/fonts/liberation-fonts/LICENSE
+STEAMPIPE_FIXUPS_PY := $(SRCDIR)/steampipe_fixups.py
 
 GECKO_VER := 2.47.2
 GECKO32_TARBALL := wine-gecko-$(GECKO_VER)-x86.tar.xz
@@ -271,8 +272,9 @@ DIST_TARGETS := $(DIST_COPY_TARGETS) $(DIST_OVR32) $(DIST_OVR64) \
                 $(DIST_GECKO32) $(DIST_GECKO64) $(DIST_WINEMONO) \
                 $(DIST_COMPAT_MANIFEST) $(DIST_LICENSE) $(DIST_TOOLMANIFEST) $(DIST_OFL_LICENSE) $(DIST_FONTS)
 
-DEPLOY_COPY_TARGETS := $(DIST_COPY_TARGETS) $(DIST_VERSION) $(DIST_LICENSE) $(DIST_TOOLMANIFEST) $(DIST_OFL_LICENSE)
-REDIST_COPY_TARGETS := $(DEPLOY_COPY_TARGETS) $(DIST_COMPAT_MANIFEST)
+BASE_COPY_TARGETS := $(DIST_COPY_TARGETS) $(DIST_VERSION) $(DIST_LICENSE) $(DIST_TOOLMANIFEST) $(DIST_OFL_LICENSE) $(DST_DIR)
+DEPLOY_COPY_TARGETS := $(BASE_COPY_TARGETS) $(STEAMPIPE_FIXUPS_PY)
+REDIST_COPY_TARGETS := $(BASE_COPY_TARGETS) $(DIST_COMPAT_MANIFEST)
 
 $(DIST_LICENSE): $(LICENSE)
 	cp -a $< $@
@@ -365,24 +367,20 @@ dist: $(DIST_TARGETS) all-dist dist_wineopenxr | $(DST_DIR)
 	echo `date '+%s'` `GIT_DIR=$(abspath $(SRCDIR)/.git) git describe --tags` > $(DIST_VERSION)
 
 deploy: dist | $(filter-out dist deploy install redist,$(MAKECMDGOALS))
-	mkdir -p $(DEPLOY_DIR) && \
-	cp -a $(DEPLOY_COPY_TARGETS) $(DEPLOY_DIR) && \
-	tar -C $(DST_DIR) -c . > $(DEPLOY_DIR)/proton_dist.tar
-	@echo "Created deployment archive at "$(DEPLOY_DIR)"/proton_dist.tar"
+	mkdir -p $(DEPLOY_DIR)
+	cp -af --no-dereference --preserve=mode,links $(DEPLOY_COPY_TARGETS) $(DEPLOY_DIR)
+	python3 $(STEAMPIPE_FIXUPS_PY) process $(DEPLOY_DIR)
 
 install: dist | $(filter-out dist deploy install redist,$(MAKECMDGOALS))
 	if [ ! -d $(STEAM_DIR) ]; then echo >&2 "!! "$(STEAM_DIR)" does not exist, cannot install"; return 1; fi
 	mkdir -p $(STEAM_DIR)/compatibilitytools.d/$(BUILD_NAME)
-	cp -rf --no-dereference --preserve=mode,links $(DST_BASE)/* $(STEAM_DIR)/compatibilitytools.d/$(BUILD_NAME)
-	cp -f $(DIST_VERSION) $(STEAM_DIR)/compatibilitytools.d/$(BUILD_NAME)/dist/
+	cp -af --no-dereference --preserve=mode,links $(DST_BASE)/* $(STEAM_DIR)/compatibilitytools.d/$(BUILD_NAME)
 	@echo "Installed Proton to "$(STEAM_DIR)/compatibilitytools.d/$(BUILD_NAME)
 	@echo "You may need to restart Steam to select this tool"
 
 redist: dist | $(filter-out dist deploy install redist,$(MAKECMDGOALS))
 	mkdir -p $(REDIST_DIR)
-	cp -a $(REDIST_COPY_TARGETS) $(REDIST_DIR)
-	tar -C $(DST_DIR) -c . | gzip -c -1 > $(REDIST_DIR)/proton_dist.tar.gz
-	@echo "Created redistribution tarball at "$(REDIST_DIR)"/proton_dist.tar.gz"
+	cp -af --no-dereference --preserve=mode,links $(REDIST_COPY_TARGETS) $(REDIST_DIR)
 
 .PHONY: module32 module64 module
 

default_pfx.py

@@ -73,6 +73,14 @@ def setup_dll_symlinks(default_pfx_dir, dist_dir):
                 os.unlink(filename)
                 make_relative_symlink(target, filename)
 
+#steampipe can't handle filenames with colons, so we remove them here
+#and restore them in the proton script
+def fixup_drive_links(default_pfx_dir):
+    for walk_dir, dirs, files in os.walk(os.path.join(default_pfx_dir, "dosdevices")):
+        for dir_ in dirs:
+            if ":" in dir_:
+                os.remove(os.path.join(walk_dir, dir_))
+
 def make_default_pfx(default_pfx_dir, dist_dir, runtime):
     local_env = dict(os.environ)
 
@@ -96,6 +104,7 @@ def make_default_pfx(default_pfx_dir, dist_dir, runtime):
 
         env=local_env, check=True)
     setup_dll_symlinks(default_pfx_dir, dist_dir)
+    fixup_drive_links(default_pfx_dir)
 
 if __name__ == '__main__':
     import sys

docker/Makefile

@@ -1,4 +1,4 @@
-STEAMRT_VERSION = 0.20210126.1
+STEAMRT_VERSION = 0.20210505.0
 STEAMRT_URLBASE = registry.gitlab.steamos.cloud
 
 PROTONSDK_URLBASE = $(STEAMRT_URLBASE)/proton/soldier/sdk

docker/proton.Dockerfile.in

@@ -52,9 +52,14 @@ RUN bash -c 'mkdir -p /usr/lib/ccache && ls /usr/bin/{,*-}{cc,c++,gcc,g++}{,-[0-
 ENV PATH=/usr/lib/ccache:$PATH
 
 RUN apt-get install -y \
-  tini \
+  libxpresent-dev \
+  libxpresent-dev:i386 \
+  python3-pefile \
+  libcapstone-dev \
+  libcapstone-dev:i386 \
 && rm -rf /opt/usr/share/doc /opt/usr/share/info /opt/usr/share/man \
 && rm -rf /var/lib/apt/lists/*
+
 ENTRYPOINT ["/usr/bin/tini-static", "-s", "-g", "--"]
 
 CMD ["/bin/bash"]

dxvk

@@ -1 +1 @@
-Subproject commit 2a98217b2d61b1fcaa3523b5edf1d81693a085dd
+Subproject commit c2d89d2b2b42a1b1acd3ba0dc6a6b36730a7af08

proton

@@ -280,14 +280,14 @@ def set_dir_casefold_bit(dir_path):
 class Proton:
     def __init__(self, base_dir):
         self.base_dir = base_dir + "/"
-        self.dist_dir = self.path("dist/")
-        self.bin_dir = self.path("dist/bin/")
-        self.lib_dir = self.path("dist/lib/")
-        self.lib64_dir = self.path("dist/lib64/")
-        self.fonts_dir = self.path("dist/share/fonts/")
-        self.wine_fonts_dir = self.path("dist/share/wine/fonts/")
+        self.dist_dir = self.path("files/")
+        self.bin_dir = self.path("files/bin/")
+        self.lib_dir = self.path("files/lib/")
+        self.lib64_dir = self.path("files/lib64/")
+        self.fonts_dir = self.path("files/share/fonts/")
+        self.wine_fonts_dir = self.path("files/share/wine/fonts/")
         self.version_file = self.path("version")
-        self.default_pfx_dir = self.path("dist/share/default_pfx/")
+        self.default_pfx_dir = self.path("files/share/default_pfx/")
         self.user_settings_file = self.path("user_settings.py")
         self.wine_bin = self.bin_dir + "wine"
         self.wineserver_bin = self.bin_dir + "wineserver"
@@ -296,28 +296,34 @@ class Proton:
     def path(self, d):
         return self.base_dir + d
 
-    def need_tarball_extraction(self):
-        '''Checks if the proton_dist tarball archive must be extracted. Returns true if extraction is needed, false otherwise'''
-        return not os.path.exists(self.dist_dir) or \
-            not os.path.exists(self.path("dist/version")) or \
-            not filecmp.cmp(self.version_file, self.path("dist/version"))
+    def cleanup_legacy_dist(self):
+        old_dist_dir = self.path("dist/")
+        if os.path.exists(old_dist_dir):
+            with self.dist_lock:
+                if os.path.exists(old_dist_dir):
+                    shutil.rmtree(old_dist_dir)
 
-    def extract_tarball(self):
-        with self.dist_lock:
-            if self.need_tarball_extraction():
-                if os.path.exists(self.dist_dir):
-                    shutil.rmtree(self.dist_dir)
-                tar = None
-                for sf in ["", ".xz", ".bz2", ".gz"]:
-                    if os.path.exists(self.path("proton_dist.tar" + sf)):
-                        tar = tarfile.open(self.path("proton_dist.tar" + sf), mode="r:*")
-                        break
-                if not tar:
-                    log("No proton_dist tarball??")
-                    sys.exit(1)
-                tar.extractall(path=self.dist_dir)
-                tar.close()
-                try_copy(self.version_file, self.dist_dir)
+    def do_steampipe_fixups(self):
+        fixups_json = self.path("steampipe_fixups.json")
+        fixups_mtime = self.path("files/steampipe_fixups_mtime")
+
+        if os.path.exists(fixups_json):
+            with self.dist_lock:
+                import steampipe_fixups
+
+                current_fixup_mtime = None
+                if os.path.exists(fixups_mtime):
+                    with open(fixups_mtime, "r") as f:
+                        current_fixup_mtime = f.readline().strip()
+
+                new_fixup_mtime = getmtimestr(fixups_json)
+
+                if current_fixup_mtime != new_fixup_mtime:
+                    result_code = steampipe_fixups.do_restore(self.base_dir, fixups_json)
+
+                    if result_code == 0:
+                        with open(fixups_mtime, "w") as f:
+                            f.write(new_fixup_mtime + "\n")
 
     def missing_default_prefix(self):
         '''Check if the default prefix dir is missing. Returns true if missing, false if present'''
@@ -648,6 +654,12 @@ class CompatData:
 
             self.migrate_user_paths()
 
+            if not os.path.lexists(self.prefix_dir + "/dosdevices/c:"):
+                os.symlink("../drive_c", self.prefix_dir + "/dosdevices/c:")
+
+            if not os.path.lexists(self.prefix_dir + "/dosdevices/z:"):
+                os.symlink("/", self.prefix_dir + "/dosdevices/z:")
+
             # collect configuration info
             steamdir = os.environ["STEAM_COMPAT_CLIENT_INSTALL_PATH"]
 
@@ -1249,8 +1261,8 @@ if __name__ == "__main__":
 
     g_proton = Proton(os.path.dirname(sys.argv[0]))
 
-    if g_proton.need_tarball_extraction():
-        g_proton.extract_tarball()
+    g_proton.cleanup_legacy_dist()
+    g_proton.do_steampipe_fixups()
 
     g_compatdata = CompatData(os.environ["STEAM_COMPAT_DATA_PATH"])
 

steampipe_fixups.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+#Steampipe doesn't support certain unix-y things which may be required by
+#native Linux applications. This file will process a directory of Linux files
+#and store the file properties into a manifest file. After a round trip through
+#Steampipe, the original file properties can be restored using this same
+#script.
+
+import json
+import os
+import secrets
+import stat
+
+DEFAULT_MANIFEST_NAME = "steampipe_fixups.json"
+
+def usage():
+    print("Usage:")
+    print("\t" + sys.argv[0] + "\tprepare\t<path to directory to process>\t[manifest output file]")
+    print("\t\tProcess the given path and output the manifest file to the given path, or <path/" + DEFAULT_MANIFEST_NAME + "> if unspecified.")
+    print("")
+    print("\t" + sys.argv[0] + "\trestore\t<path to directory to process>\t[manifest file]")
+    print("\t\tRestore the given path using the manifest file, or <path/" + DEFAULT_MANIFEST_NAME + "> if unspecified.")
+
+empty_dirs = []
+no_write_paths = []
+
+def canonicalize(path, prefix):
+    return path.replace(prefix, "", 1).lstrip('/')
+
+def process_dir(path):
+    for root, subdirs, files in os.walk(path):
+        if len(subdirs) == 0 and len(files) == 0:
+            empty_dirs.append(canonicalize(root, path))
+
+        for file_ in files:
+            this_file = os.path.join(root, file_)
+            stat_result = os.lstat(this_file)
+            if (stat_result.st_mode & stat.S_IWUSR) == 0:
+                no_write_paths.append(canonicalize(this_file, path))
+
+    return 0
+
+def write_manifest(manifest):
+    out = open(manifest, "w")
+    json.dump(
+        {
+            "id": str(secrets.randbits(32)), #we need steampipe to update this file for every build
+            "empty_dirs": empty_dirs,
+            "no_write_paths": no_write_paths,
+        },
+        out,
+        indent = 4,
+        sort_keys = True
+    )
+    return 0
+
+def do_process(path, manifest):
+    if os.path.exists(manifest):
+        os.remove(manifest)
+
+    ret = process_dir(path)
+    if ret != 0:
+        return ret
+
+    #output should be deterministic
+    empty_dirs.sort()
+    no_write_paths.sort()
+
+    ret = write_manifest(manifest)
+    if ret != 0:
+        return ret
+
+    return 0
+
+def do_restore(path, manifest):
+    loaded = json.load(open(manifest, "r"))
+
+    empty_dirs = loaded["empty_dirs"]
+    no_write_paths = loaded["no_write_paths"]
+
+    for empty_dir in empty_dirs:
+        try:
+            os.makedirs(os.path.join(path, empty_dir))
+        except OSError:
+            #already exists
+            pass
+
+    for file_ in no_write_paths:
+        this_file = os.path.join(path, file_)
+        stat_result = os.lstat(this_file)
+        os.chmod(this_file,
+                stat_result.st_mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
+
+    return 0
+
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) < 3 or len(sys.argv) > 4:
+        usage()
+        sys.exit(1)
+
+    verb = sys.argv[1]
+    path = sys.argv[2]
+    if len(sys.argv) >= 4:
+        manifest = sys.argv[3]
+    else:
+        manifest = os.path.join(path, DEFAULT_MANIFEST_NAME)
+
+    if verb == "process":
+        sys.exit(do_process(path, manifest))
+
+    if verb == "restore":
+        sys.exit(do_restore(path, manifest))
+
+    usage()
+    sys.exit(1)

vkd3d-proton

@@ -1 +1 @@
-Subproject commit cafe99e223296e2a44a2b3d1f6e99e5d49580de1
+Subproject commit 7ff3ef265480f0b4527f0a7bbfc5098bd4d0b173

wine

@@ -1 +1 @@
-Subproject commit 3083b57f002a84b858ca6a093e90527665f61d94
+Subproject commit 657daf49680e03e8b0310ee5a9d8860e1e7b5f07