[PATCH] Escape percent-signs in filename when fetching from mirrors

classic Classic list List threaded Threaded
5 messages Options
Reply | Threaded
Open this post in threaded view
|

[PATCH] Escape percent-signs in filename when fetching from mirrors

Mike Gilbert-2
Bug: https://bugs.gentoo.org/719810
Signed-off-by: Mike Gilbert <[hidden email]>
---
 lib/portage/package/ebuild/fetch.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
index 28e7caf53..47c3ad28f 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -26,6 +26,11 @@ try:
 except ImportError:
  from urlparse import urlparse
 
+try:
+ from urllib.parse import quote as urlquote
+except ImportError:
+ from urllib import quote as urlquote
+
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
  'portage.package.ebuild.config:check_config_instance,config',
@@ -351,7 +356,7 @@ _size_suffix_map = {
 
 class FlatLayout(object):
  def get_path(self, filename):
- return filename
+ return urlquote(filename)
 
  def get_filenames(self, distdir):
  for dirpath, dirnames, filenames in os.walk(distdir,
@@ -382,7 +387,7 @@ class FilenameHashLayout(object):
  c = c // 4
  ret += fnhash[:c] + '/'
  fnhash = fnhash[c:]
- return ret + filename
+ return ret + urlquote(filename)
 
  def get_filenames(self, distdir):
  pattern = ''
--
2.27.0.rc2


Reply | Threaded
Open this post in threaded view
|

Re: [PATCH] Escape percent-signs in filename when fetching from mirrors

Zac Medico-2
On 5/30/20 8:26 PM, Mike Gilbert wrote:

> Bug: https://bugs.gentoo.org/719810
> Signed-off-by: Mike Gilbert <[hidden email]>
> ---
>  lib/portage/package/ebuild/fetch.py | 9 +++++++--
>  1 file changed, 7 insertions(+), 2 deletions(-)
>
> diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
> index 28e7caf53..47c3ad28f 100644
> --- a/lib/portage/package/ebuild/fetch.py
> +++ b/lib/portage/package/ebuild/fetch.py
> @@ -26,6 +26,11 @@ try:
>  except ImportError:
>   from urlparse import urlparse
>  
> +try:
> + from urllib.parse import quote as urlquote
> +except ImportError:
> + from urllib import quote as urlquote
> +
>  import portage
>  portage.proxy.lazyimport.lazyimport(globals(),
>   'portage.package.ebuild.config:check_config_instance,config',
> @@ -351,7 +356,7 @@ _size_suffix_map = {
>  
>  class FlatLayout(object):
>   def get_path(self, filename):
> - return filename
> + return urlquote(filename)
>  
>   def get_filenames(self, distdir):
>   for dirpath, dirnames, filenames in os.walk(distdir,
> @@ -382,7 +387,7 @@ class FilenameHashLayout(object):
>   c = c // 4
>   ret += fnhash[:c] + '/'
>   fnhash = fnhash[c:]
> - return ret + filename
> + return ret + urlquote(filename)
>  
>   def get_filenames(self, distdir):
>   pattern = ''
>
In order to ensure that the filename will not have an extra level of quoting
in the mirror path, we'll have to do something like this wherever we
translate a uri in SRC_URI to a filename:

diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
index 08af17bcd..99d8ab720 100644
--- a/lib/portage/dbapi/porttree.py
+++ b/lib/portage/dbapi/porttree.py
@@ -1547,7 +1547,7 @@ def _parse_uri_map(cpv, metadata, use=None):
                        myuris.pop()
                        distfile = myuris.pop()
                else:
-                       distfile = os.path.basename(uri)
+                       distfile = unquote(os.path.basename(uri))
                        if not distfile:
                                raise portage.exception.InvalidDependString(
                                        ("getFetchMap(): '%s' SRC_URI has no file " + \

--
Thanks,
Zac


signature.asc (1000 bytes) Download Attachment
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH] Escape percent-signs in filename when fetching from mirrors

Ulrich Mueller-2
>>>>> On Sun, 31 May 2020, Zac Medico wrote:

> In order to ensure that the filename will not have an extra level of quoting
> in the mirror path, we'll have to do something like this wherever we
> translate a uri in SRC_URI to a filename:

> diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py
> index 08af17bcd..99d8ab720 100644
> --- a/lib/portage/dbapi/porttree.py
> +++ b/lib/portage/dbapi/porttree.py
> @@ -1547,7 +1547,7 @@ def _parse_uri_map(cpv, metadata, use=None):
>                         myuris.pop()
>                         distfile = myuris.pop()
>                 else:
> -                       distfile = os.path.basename(uri)
> +                       distfile = unquote(os.path.basename(uri))
>                         if not distfile:
>                                 raise portage.exception.InvalidDependString(
>                                         ("getFetchMap(): '%s' SRC_URI has no file " + \
I think that unescaping is correct, but you'll need a subsequent sanity
check to make sure that the resulting filename doesn't contain any
characters that aren't allowed in the Manifest.

Namely, "excluding the backwards slash (\) and characters classified as
control characters or as whitespace" (GLEP 74).

Ulrich

signature.asc (517 bytes) Download Attachment
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH] Escape percent-signs in filename when fetching from mirrors

Zac Medico-2
In reply to this post by Mike Gilbert-2
On 5/30/20 8:26 PM, Mike Gilbert wrote:

> Bug: https://bugs.gentoo.org/719810
> Signed-off-by: Mike Gilbert <[hidden email]>
> ---
>  lib/portage/package/ebuild/fetch.py | 9 +++++++--
>  1 file changed, 7 insertions(+), 2 deletions(-)
>
> diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
> index 28e7caf53..47c3ad28f 100644
> --- a/lib/portage/package/ebuild/fetch.py
> +++ b/lib/portage/package/ebuild/fetch.py
> @@ -26,6 +26,11 @@ try:
>  except ImportError:
>   from urlparse import urlparse
>  
> +try:
> + from urllib.parse import quote as urlquote
> +except ImportError:
> + from urllib import quote as urlquote
> +
>  import portage
>  portage.proxy.lazyimport.lazyimport(globals(),
>   'portage.package.ebuild.config:check_config_instance,config',
> @@ -351,7 +356,7 @@ _size_suffix_map = {
>  
>  class FlatLayout(object):
>   def get_path(self, filename):
> - return filename
> + return urlquote(filename)
>  
>   def get_filenames(self, distdir):
>   for dirpath, dirnames, filenames in os.walk(distdir,
> @@ -382,7 +387,7 @@ class FilenameHashLayout(object):
>   c = c // 4
>   ret += fnhash[:c] + '/'
>   fnhash = fnhash[c:]
> - return ret + filename
> + return ret + urlquote(filename)
>  
>   def get_filenames(self, distdir):
>   pattern = ''
>
Please go ahead an merge, since SRC_URI arrows allow use to neglect
unquoting as discussed in
https://archives.gentoo.org/gentoo-portage-dev/message/176d5e6f38b346be760d05f9c6e72e02.
--
Thanks,
Zac


signature.asc (1000 bytes) Download Attachment
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH] Escape percent-signs in filename when fetching from mirrors

Mike Gilbert-2
On Sun, May 31, 2020 at 4:20 PM Zac Medico <[hidden email]> wrote:

>
> On 5/30/20 8:26 PM, Mike Gilbert wrote:
> > Bug: https://bugs.gentoo.org/719810
> > Signed-off-by: Mike Gilbert <[hidden email]>
> > ---
> >  lib/portage/package/ebuild/fetch.py | 9 +++++++--
> >  1 file changed, 7 insertions(+), 2 deletions(-)
> >
> > diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
> > index 28e7caf53..47c3ad28f 100644
> > --- a/lib/portage/package/ebuild/fetch.py
> > +++ b/lib/portage/package/ebuild/fetch.py
> > @@ -26,6 +26,11 @@ try:
> >  except ImportError:
> >       from urlparse import urlparse
> >
> > +try:
> > +     from urllib.parse import quote as urlquote
> > +except ImportError:
> > +     from urllib import quote as urlquote
> > +
> >  import portage
> >  portage.proxy.lazyimport.lazyimport(globals(),
> >       'portage.package.ebuild.config:check_config_instance,config',
> > @@ -351,7 +356,7 @@ _size_suffix_map = {
> >
> >  class FlatLayout(object):
> >       def get_path(self, filename):
> > -             return filename
> > +             return urlquote(filename)
> >
> >       def get_filenames(self, distdir):
> >               for dirpath, dirnames, filenames in os.walk(distdir,
> > @@ -382,7 +387,7 @@ class FilenameHashLayout(object):
> >                       c = c // 4
> >                       ret += fnhash[:c] + '/'
> >                       fnhash = fnhash[c:]
> > -             return ret + filename
> > +             return ret + urlquote(filename)
> >
> >       def get_filenames(self, distdir):
> >               pattern = ''
> >
>
> Please go ahead an merge, since SRC_URI arrows allow use to neglect
> unquoting as discussed in
> https://archives.gentoo.org/gentoo-portage-dev/message/176d5e6f38b346be760d05f9c6e72e02.

I ended up reverting this; I did not realize emirrordist shared the
same code. A new patch is on its way.