[PATCH] repoman: add --include-profiles=PROFILES

classic Classic list List threaded Threaded
7 messages Options
Reply | Threaded
Open this post in threaded view
|

[PATCH] repoman: add --include-profiles=PROFILES

Sergei Trofimovich-6
repoman slows down ~linearly with amount of profiles being scanned.
In case of amd64 we have 28 stable profiles.

To speed up processing and fit into time budged of various CIs we can
split the work across different processes that handle different profiles.

Example benchmark on ::haskell overlay:
    $ ./repoman full --include-arches=amd64
    ~65 minutes
    $ ./repoman full --include-profiles=default/linux/amd64/17.0
    ~4 minutes
This allows for a crude sharding of work across processes and allows for
cheap tree-wide scans for early failures.

Bug: https://bugs.gentoo.org/700456
Signed-off-by: Sergei Trofimovich <[hidden email]>
---
 repoman/lib/repoman/actions.py                      | 4 ++++
 repoman/lib/repoman/argparser.py                    | 7 +++++++
 repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++-
 repoman/lib/repoman/modules/scan/depend/profile.py  | 9 +++++++--
 repoman/lib/repoman/scanner.py                      | 5 +++++
 repoman/man/repoman.1                               | 4 ++++
 6 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py
index 1c9989a72..92d4d4e94 100644
--- a/repoman/lib/repoman/actions.py
+++ b/repoman/lib/repoman/actions.py
@@ -412,6 +412,10 @@ the whole commit message to abort.
  report_options.append(
  "--include-arches=\"%s\"" %
  " ".join(sorted(self.scanner.include_arches)))
+ if self.scanner.include_profiles is not None:
+ report_options.append(
+ "--include-profiles=\"%s\"" %
+ " ".join(sorted(self.scanner.include_profiles)))
 
  if portage_version is None:
  sys.stderr.write("Failed to insert portage version in message!\n")
diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py
index fa0e6ff90..670a0e91d 100644
--- a/repoman/lib/repoman/argparser.py
+++ b/repoman/lib/repoman/argparser.py
@@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts):
  'A space separated list of arches used to '
  'filter the selection of profiles for dependency checks'))
 
+ parser.add_argument(
+ '--include-profiles',
+ dest='include_profiles', metavar='PROFILES', action='append',
+ help=(
+ 'A space separated list of profiles used to '
+ 'define the selection of profiles for dependency checks'))
+
  parser.add_argument(
  '-d', '--include-dev', dest='include_dev', action='store_true',
  default=False,
diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py
index c3cc0ddeb..9068760bb 100644
--- a/repoman/lib/repoman/modules/scan/depend/__init__.py
+++ b/repoman/lib/repoman/modules/scan/depend/__init__.py
@@ -19,7 +19,8 @@ module_spec = {
  'func_desc': {
  },
  'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options',
- 'repo_metadata', 'repo_settings', 'include_arches', 'caches',
+ 'repo_metadata', 'repo_settings', 'include_arches',
+ 'include_profiles', 'caches',
  'repoman_incrementals', 'env', 'have', 'dev_keywords'
  ],
  'func_kwargs': {
diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py
index d980f4eca..0b1d74483 100644
--- a/repoman/lib/repoman/modules/scan/depend/profile.py
+++ b/repoman/lib/repoman/modules/scan/depend/profile.py
@@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase):
  @param options: cli options
  @param repo_settings: repository settings instance
  @param include_arches: set
+ @param include_profiles: set
  @param caches: dictionary of our caches
  @param repoman_incrementals: tuple
  @param env: the environment
@@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase):
  self.options = kwargs.get('options')
  self.repo_settings = kwargs.get('repo_settings')
  self.include_arches = kwargs.get('include_arches')
+ self.include_profiles = kwargs.get('include_profiles')
  self.caches = kwargs.get('caches')
  self.repoman_incrementals = kwargs.get('repoman_incrementals')
  self.env = kwargs.get('env')
@@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase):
  if arch not in self.include_arches:
  continue
 
- relevant_profiles.extend(
- (keyword, groups, prof) for prof in self.profiles[arch])
+ for prof in self.profiles[arch]:
+ if self.include_profiles is not None:
+ if prof not in self.include_profiles:
+ continue
+ relevant_profiles.append((keyword, groups, prof))
 
  relevant_profiles.sort(key=sort_key)
 
diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py
index 1b3242a51..06234b0ad 100644
--- a/repoman/lib/repoman/scanner.py
+++ b/repoman/lib/repoman/scanner.py
@@ -164,6 +164,10 @@ class Scanner(object):
  if self.options.include_arches:
  self.include_arches = set()
  self.include_arches.update(*[x.split() for x in self.options.include_arches])
+ self.include_profiles = None
+ if self.options.include_profiles:
+ self.include_profiles = set()
+ self.include_profiles.update(*[x.split() for x in self.options.include_profiles])
 
  # Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and
  # running `svn status` in every package dir will be too expensive.
@@ -190,6 +194,7 @@ class Scanner(object):
  "repo_metadata": self.repo_metadata,
  "profiles": profiles,
  "include_arches": self.include_arches,
+ "include_profiles": self.include_profiles,
  "caches": self.caches,
  "repoman_incrementals": self.repoman_incrementals,
  "env": self.env,
diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1
index 766146f57..f33fb6098 100644
--- a/repoman/man/repoman.1
+++ b/repoman/man/repoman.1
@@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode)
 A space separated list of arches used to filter the selection of
 profiles for dependency checks.
 .TP
+.BR "\-\-include\-profiles " PROFILES
+A space separated list of profiles used to
+define the selection of profiles for dependency checks.
+.TP
 \fB\-d\fR, \fB\-\-include\-dev\fR
 Include dev profiles in dependency checks.
 .TP
--
2.24.0


Reply | Threaded
Open this post in threaded view
|

Re: [PATCH] repoman: add --include-profiles=PROFILES

Zac Medico-2
On 11/18/19 4:21 PM, Sergei Trofimovich wrote:

> repoman slows down ~linearly with amount of profiles being scanned.
> In case of amd64 we have 28 stable profiles.
>
> To speed up processing and fit into time budged of various CIs we can
> split the work across different processes that handle different profiles.
>
> Example benchmark on ::haskell overlay:
>     $ ./repoman full --include-arches=amd64
>     ~65 minutes
>     $ ./repoman full --include-profiles=default/linux/amd64/17.0
>     ~4 minutes
> This allows for a crude sharding of work across processes and allows for
> cheap tree-wide scans for early failures.
>
> Bug: https://bugs.gentoo.org/700456
> Signed-off-by: Sergei Trofimovich <[hidden email]>
> ---
>  repoman/lib/repoman/actions.py                      | 4 ++++
>  repoman/lib/repoman/argparser.py                    | 7 +++++++
>  repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++-
>  repoman/lib/repoman/modules/scan/depend/profile.py  | 9 +++++++--
>  repoman/lib/repoman/scanner.py                      | 5 +++++
>  repoman/man/repoman.1                               | 4 ++++
>  6 files changed, 29 insertions(+), 3 deletions(-)
>
> diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py
> index 1c9989a72..92d4d4e94 100644
> --- a/repoman/lib/repoman/actions.py
> +++ b/repoman/lib/repoman/actions.py
> @@ -412,6 +412,10 @@ the whole commit message to abort.
>   report_options.append(
>   "--include-arches=\"%s\"" %
>   " ".join(sorted(self.scanner.include_arches)))
> + if self.scanner.include_profiles is not None:
> + report_options.append(
> + "--include-profiles=\"%s\"" %
> + " ".join(sorted(self.scanner.include_profiles)))
>  
>   if portage_version is None:
>   sys.stderr.write("Failed to insert portage version in message!\n")
> diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py
> index fa0e6ff90..670a0e91d 100644
> --- a/repoman/lib/repoman/argparser.py
> +++ b/repoman/lib/repoman/argparser.py
> @@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts):
>   'A space separated list of arches used to '
>   'filter the selection of profiles for dependency checks'))
>  
> + parser.add_argument(
> + '--include-profiles',
> + dest='include_profiles', metavar='PROFILES', action='append',
> + help=(
> + 'A space separated list of profiles used to '
> + 'define the selection of profiles for dependency checks'))
> +
>   parser.add_argument(
>   '-d', '--include-dev', dest='include_dev', action='store_true',
>   default=False,
> diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py
> index c3cc0ddeb..9068760bb 100644
> --- a/repoman/lib/repoman/modules/scan/depend/__init__.py
> +++ b/repoman/lib/repoman/modules/scan/depend/__init__.py
> @@ -19,7 +19,8 @@ module_spec = {
>   'func_desc': {
>   },
>   'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options',
> - 'repo_metadata', 'repo_settings', 'include_arches', 'caches',
> + 'repo_metadata', 'repo_settings', 'include_arches',
> + 'include_profiles', 'caches',
>   'repoman_incrementals', 'env', 'have', 'dev_keywords'
>   ],
>   'func_kwargs': {
> diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py
> index d980f4eca..0b1d74483 100644
> --- a/repoman/lib/repoman/modules/scan/depend/profile.py
> +++ b/repoman/lib/repoman/modules/scan/depend/profile.py
> @@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase):
>   @param options: cli options
>   @param repo_settings: repository settings instance
>   @param include_arches: set
> + @param include_profiles: set
>   @param caches: dictionary of our caches
>   @param repoman_incrementals: tuple
>   @param env: the environment
> @@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase):
>   self.options = kwargs.get('options')
>   self.repo_settings = kwargs.get('repo_settings')
>   self.include_arches = kwargs.get('include_arches')
> + self.include_profiles = kwargs.get('include_profiles')
>   self.caches = kwargs.get('caches')
>   self.repoman_incrementals = kwargs.get('repoman_incrementals')
>   self.env = kwargs.get('env')
> @@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase):
>   if arch not in self.include_arches:
>   continue
>  
> - relevant_profiles.extend(
> - (keyword, groups, prof) for prof in self.profiles[arch])
> + for prof in self.profiles[arch]:
> + if self.include_profiles is not None:
> + if prof not in self.include_profiles:
> + continue
> + relevant_profiles.append((keyword, groups, prof))
>  
>   relevant_profiles.sort(key=sort_key)
>  
> diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py
> index 1b3242a51..06234b0ad 100644
> --- a/repoman/lib/repoman/scanner.py
> +++ b/repoman/lib/repoman/scanner.py
> @@ -164,6 +164,10 @@ class Scanner(object):
>   if self.options.include_arches:
>   self.include_arches = set()
>   self.include_arches.update(*[x.split() for x in self.options.include_arches])
> + self.include_profiles = None
> + if self.options.include_profiles:
> + self.include_profiles = set()
> + self.include_profiles.update(*[x.split() for x in self.options.include_profiles])
>  
>   # Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and
>   # running `svn status` in every package dir will be too expensive.
> @@ -190,6 +194,7 @@ class Scanner(object):
>   "repo_metadata": self.repo_metadata,
>   "profiles": profiles,
>   "include_arches": self.include_arches,
> + "include_profiles": self.include_profiles,
>   "caches": self.caches,
>   "repoman_incrementals": self.repoman_incrementals,
>   "env": self.env,
> diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1
> index 766146f57..f33fb6098 100644
> --- a/repoman/man/repoman.1
> +++ b/repoman/man/repoman.1
> @@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode)
>  A space separated list of arches used to filter the selection of
>  profiles for dependency checks.
>  .TP
> +.BR "\-\-include\-profiles " PROFILES
> +A space separated list of profiles used to
> +define the selection of profiles for dependency checks.
> +.TP
>  \fB\-d\fR, \fB\-\-include\-dev\fR
>  Include dev profiles in dependency checks.
>  .TP
>
Looks good. Please merge.
--
Thanks,
Zac


signature.asc (1000 bytes) Download Attachment
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH] repoman: add --include-profiles=PROFILES

Sergei Trofimovich-6
On Mon, 18 Nov 2019 16:45:58 -0800
Zac Medico <[hidden email]> wrote:

> On 11/18/19 4:21 PM, Sergei Trofimovich wrote:
> > repoman slows down ~linearly with amount of profiles being scanned.
> > In case of amd64 we have 28 stable profiles.
> >
> > To speed up processing and fit into time budged of various CIs we can
> > split the work across different processes that handle different profiles.
> >
> > Example benchmark on ::haskell overlay:
> >     $ ./repoman full --include-arches=amd64
> >     ~65 minutes
> >     $ ./repoman full --include-profiles=default/linux/amd64/17.0
> >     ~4 minutes
> > This allows for a crude sharding of work across processes and allows for
> > cheap tree-wide scans for early failures.
> >
> > Bug: https://bugs.gentoo.org/700456
> > Signed-off-by: Sergei Trofimovich <[hidden email]>
> > ---
> >  repoman/lib/repoman/actions.py                      | 4 ++++
> >  repoman/lib/repoman/argparser.py                    | 7 +++++++
> >  repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++-
> >  repoman/lib/repoman/modules/scan/depend/profile.py  | 9 +++++++--
> >  repoman/lib/repoman/scanner.py                      | 5 +++++
> >  repoman/man/repoman.1                               | 4 ++++
> >  6 files changed, 29 insertions(+), 3 deletions(-)
> >
> > diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py
> > index 1c9989a72..92d4d4e94 100644
> > --- a/repoman/lib/repoman/actions.py
> > +++ b/repoman/lib/repoman/actions.py
> > @@ -412,6 +412,10 @@ the whole commit message to abort.
> >   report_options.append(
> >   "--include-arches=\"%s\"" %
> >   " ".join(sorted(self.scanner.include_arches)))
> > + if self.scanner.include_profiles is not None:
> > + report_options.append(
> > + "--include-profiles=\"%s\"" %
> > + " ".join(sorted(self.scanner.include_profiles)))
> >  
> >   if portage_version is None:
> >   sys.stderr.write("Failed to insert portage version in message!\n")
> > diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py
> > index fa0e6ff90..670a0e91d 100644
> > --- a/repoman/lib/repoman/argparser.py
> > +++ b/repoman/lib/repoman/argparser.py
> > @@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts):
> >   'A space separated list of arches used to '
> >   'filter the selection of profiles for dependency checks'))
> >  
> > + parser.add_argument(
> > + '--include-profiles',
> > + dest='include_profiles', metavar='PROFILES', action='append',
> > + help=(
> > + 'A space separated list of profiles used to '
> > + 'define the selection of profiles for dependency checks'))
> > +
> >   parser.add_argument(
> >   '-d', '--include-dev', dest='include_dev', action='store_true',
> >   default=False,
> > diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py
> > index c3cc0ddeb..9068760bb 100644
> > --- a/repoman/lib/repoman/modules/scan/depend/__init__.py
> > +++ b/repoman/lib/repoman/modules/scan/depend/__init__.py
> > @@ -19,7 +19,8 @@ module_spec = {
> >   'func_desc': {
> >   },
> >   'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options',
> > - 'repo_metadata', 'repo_settings', 'include_arches', 'caches',
> > + 'repo_metadata', 'repo_settings', 'include_arches',
> > + 'include_profiles', 'caches',
> >   'repoman_incrementals', 'env', 'have', 'dev_keywords'
> >   ],
> >   'func_kwargs': {
> > diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py
> > index d980f4eca..0b1d74483 100644
> > --- a/repoman/lib/repoman/modules/scan/depend/profile.py
> > +++ b/repoman/lib/repoman/modules/scan/depend/profile.py
> > @@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase):
> >   @param options: cli options
> >   @param repo_settings: repository settings instance
> >   @param include_arches: set
> > + @param include_profiles: set
> >   @param caches: dictionary of our caches
> >   @param repoman_incrementals: tuple
> >   @param env: the environment
> > @@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase):
> >   self.options = kwargs.get('options')
> >   self.repo_settings = kwargs.get('repo_settings')
> >   self.include_arches = kwargs.get('include_arches')
> > + self.include_profiles = kwargs.get('include_profiles')
> >   self.caches = kwargs.get('caches')
> >   self.repoman_incrementals = kwargs.get('repoman_incrementals')
> >   self.env = kwargs.get('env')
> > @@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase):
> >   if arch not in self.include_arches:
> >   continue
> >  
> > - relevant_profiles.extend(
> > - (keyword, groups, prof) for prof in self.profiles[arch])
> > + for prof in self.profiles[arch]:
> > + if self.include_profiles is not None:
> > + if prof not in self.include_profiles:
> > + continue
> > + relevant_profiles.append((keyword, groups, prof))
> >  
> >   relevant_profiles.sort(key=sort_key)
> >  
> > diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py
> > index 1b3242a51..06234b0ad 100644
> > --- a/repoman/lib/repoman/scanner.py
> > +++ b/repoman/lib/repoman/scanner.py
> > @@ -164,6 +164,10 @@ class Scanner(object):
> >   if self.options.include_arches:
> >   self.include_arches = set()
> >   self.include_arches.update(*[x.split() for x in self.options.include_arches])
> > + self.include_profiles = None
> > + if self.options.include_profiles:
> > + self.include_profiles = set()
> > + self.include_profiles.update(*[x.split() for x in self.options.include_profiles])
> >  
> >   # Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and
> >   # running `svn status` in every package dir will be too expensive.
> > @@ -190,6 +194,7 @@ class Scanner(object):
> >   "repo_metadata": self.repo_metadata,
> >   "profiles": profiles,
> >   "include_arches": self.include_arches,
> > + "include_profiles": self.include_profiles,
> >   "caches": self.caches,
> >   "repoman_incrementals": self.repoman_incrementals,
> >   "env": self.env,
> > diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1
> > index 766146f57..f33fb6098 100644
> > --- a/repoman/man/repoman.1
> > +++ b/repoman/man/repoman.1
> > @@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode)
> >  A space separated list of arches used to filter the selection of
> >  profiles for dependency checks.
> >  .TP
> > +.BR "\-\-include\-profiles " PROFILES
> > +A space separated list of profiles used to
> > +define the selection of profiles for dependency checks.
> > +.TP
> >  \fB\-d\fR, \fB\-\-include\-dev\fR
> >  Include dev profiles in dependency checks.
> >  .TP
> >  
>
> Looks good. Please merge.
> --
> Thanks,
> Zac
>
Thank you!

Does not look like I have permissions to do it:

"""
$ cat .git/config
[core]
        repositoryformatversion = 0
        filemode = true
        bare = false
        logallrefupdates = true
[remote "origin"]
        url = git+ssh://[hidden email]/proj/portage.git
        fetch = +refs/heads/*:refs/remotes/origin/*
[branch "master"]
        remote = origin
        merge = refs/heads/master
$ LANG=C git push origin master

FATAL -- ACCESS DENIED
Repo            proj/portage
User            [hidden email]
Stage           Before git was called
Operation       Repo write

FATAL: W any proj/portage [hidden email] DENIED by fallthru
(or you mis-spelled the reponame)
fatal: Could not read from remote repository.

Please make sure you have the correct access rights
and the repository exists.
"""

--

  Sergei

attachment0 (1000 bytes) Download Attachment
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH] repoman: add --include-profiles=PROFILES

Pacho Ramos
In reply to this post by Sergei Trofimovich-6
El mar, 19-11-2019 a las 00:21 +0000, Sergei Trofimovich escribió:

> repoman slows down ~linearly with amount of profiles being scanned.
> In case of amd64 we have 28 stable profiles.
>
> To speed up processing and fit into time budged of various CIs we can
> split the work across different processes that handle different profiles.
>
> Example benchmark on ::haskell overlay:
>     $ ./repoman full --include-arches=amd64
>     ~65 minutes
>     $ ./repoman full --include-profiles=default/linux/amd64/17.0
>     ~4 minutes
> This allows for a crude sharding of work across processes and allows for
> cheap tree-wide scans for early failures.
>
Just for knowing (as I guess there is a technical issue preventing that), why
repoman is not trying to check one profile per core in parallel by default by
itself?

Thanks a lot for the info :)

signature.asc (499 bytes) Download Attachment
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH] repoman: add --include-profiles=PROFILES

Zac Medico-2
On 11/19/19 8:34 AM, Pacho Ramos wrote:

> El mar, 19-11-2019 a las 00:21 +0000, Sergei Trofimovich escribió:
>> repoman slows down ~linearly with amount of profiles being scanned.
>> In case of amd64 we have 28 stable profiles.
>>
>> To speed up processing and fit into time budged of various CIs we can
>> split the work across different processes that handle different profiles.
>>
>> Example benchmark on ::haskell overlay:
>>     $ ./repoman full --include-arches=amd64
>>     ~65 minutes
>>     $ ./repoman full --include-profiles=default/linux/amd64/17.0
>>     ~4 minutes
>> This allows for a crude sharding of work across processes and allows for
>> cheap tree-wide scans for early failures.
>>
>
> Just for knowing (as I guess there is a technical issue preventing that), why
> repoman is not trying to check one profile per core in parallel by default by
> itself?
Some things to consider when implementing that:

* Some metadata cache backends do not work across forks (sqlite).

* You'll need a master process to aggregate serialized results from
child processes.

* Some work which is currently done only once would likely have to be
repeated in parallel child processes, for example multiple processes
would be simultaneously parsing the same metadata cache entries or even
generating metadata for the same package simultaneously.

> Thanks a lot for the info :)
--
Thanks,
Zac


signature.asc (352 bytes) Download Attachment
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH] repoman: add --include-profiles=PROFILES

Zac Medico-2
In reply to this post by Sergei Trofimovich-6
On 11/18/19 4:21 PM, Sergei Trofimovich wrote:

> diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py
> index d980f4eca..0b1d74483 100644
> --- a/repoman/lib/repoman/modules/scan/depend/profile.py
> +++ b/repoman/lib/repoman/modules/scan/depend/profile.py
> @@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase):
>   @param options: cli options
>   @param repo_settings: repository settings instance
>   @param include_arches: set
> + @param include_profiles: set
>   @param caches: dictionary of our caches
>   @param repoman_incrementals: tuple
>   @param env: the environment
> @@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase):
>   self.options = kwargs.get('options')
>   self.repo_settings = kwargs.get('repo_settings')
>   self.include_arches = kwargs.get('include_arches')
> + self.include_profiles = kwargs.get('include_profiles')
>   self.caches = kwargs.get('caches')
>   self.repoman_incrementals = kwargs.get('repoman_incrementals')
>   self.env = kwargs.get('env')
> @@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase):
>   if arch not in self.include_arches:
>   continue
>  
> - relevant_profiles.extend(
> - (keyword, groups, prof) for prof in self.profiles[arch])
> + for prof in self.profiles[arch]:
> + if self.include_profiles is not None:
> + if prof not in self.include_profiles:
Since prof is an instance of repoman.profile.ProfileDesc, you actually
have to use prof.sub_path for this containment check. That means your
benchmark skipped the profile that you intended to include.

> + continue
> + relevant_profiles.append((keyword, groups, prof))
--
Thanks,
Zac


signature.asc (1000 bytes) Download Attachment
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH] repoman: add --include-profiles=PROFILES

Zac Medico-2
In reply to this post by Sergei Trofimovich-6
On 11/18/19 11:41 PM, Sergei Trofimovich wrote:

> On Mon, 18 Nov 2019 16:45:58 -0800
> Zac Medico <[hidden email]> wrote:
>
>> On 11/18/19 4:21 PM, Sergei Trofimovich wrote:
>>> repoman slows down ~linearly with amount of profiles being scanned.
>>> In case of amd64 we have 28 stable profiles.
>>>
>>> To speed up processing and fit into time budged of various CIs we can
>>> split the work across different processes that handle different profiles.
>>>
>>> Example benchmark on ::haskell overlay:
>>>     $ ./repoman full --include-arches=amd64
>>>     ~65 minutes
>>>     $ ./repoman full --include-profiles=default/linux/amd64/17.0
>>>     ~4 minutes
>>> This allows for a crude sharding of work across processes and allows for
>>> cheap tree-wide scans for early failures.
>>>
>>> Bug: https://bugs.gentoo.org/700456
>>> Signed-off-by: Sergei Trofimovich <[hidden email]>
>>> ---
>>>  repoman/lib/repoman/actions.py                      | 4 ++++
>>>  repoman/lib/repoman/argparser.py                    | 7 +++++++
>>>  repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++-
>>>  repoman/lib/repoman/modules/scan/depend/profile.py  | 9 +++++++--
>>>  repoman/lib/repoman/scanner.py                      | 5 +++++
>>>  repoman/man/repoman.1                               | 4 ++++
>>>  6 files changed, 29 insertions(+), 3 deletions(-)
>>>
>>> diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py
>>> index 1c9989a72..92d4d4e94 100644
>>> --- a/repoman/lib/repoman/actions.py
>>> +++ b/repoman/lib/repoman/actions.py
>>> @@ -412,6 +412,10 @@ the whole commit message to abort.
>>>   report_options.append(
>>>   "--include-arches=\"%s\"" %
>>>   " ".join(sorted(self.scanner.include_arches)))
>>> + if self.scanner.include_profiles is not None:
>>> + report_options.append(
>>> + "--include-profiles=\"%s\"" %
>>> + " ".join(sorted(self.scanner.include_profiles)))
>>>  
>>>   if portage_version is None:
>>>   sys.stderr.write("Failed to insert portage version in message!\n")
>>> diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py
>>> index fa0e6ff90..670a0e91d 100644
>>> --- a/repoman/lib/repoman/argparser.py
>>> +++ b/repoman/lib/repoman/argparser.py
>>> @@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts):
>>>   'A space separated list of arches used to '
>>>   'filter the selection of profiles for dependency checks'))
>>>  
>>> + parser.add_argument(
>>> + '--include-profiles',
>>> + dest='include_profiles', metavar='PROFILES', action='append',
>>> + help=(
>>> + 'A space separated list of profiles used to '
>>> + 'define the selection of profiles for dependency checks'))
>>> +
>>>   parser.add_argument(
>>>   '-d', '--include-dev', dest='include_dev', action='store_true',
>>>   default=False,
>>> diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py
>>> index c3cc0ddeb..9068760bb 100644
>>> --- a/repoman/lib/repoman/modules/scan/depend/__init__.py
>>> +++ b/repoman/lib/repoman/modules/scan/depend/__init__.py
>>> @@ -19,7 +19,8 @@ module_spec = {
>>>   'func_desc': {
>>>   },
>>>   'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options',
>>> - 'repo_metadata', 'repo_settings', 'include_arches', 'caches',
>>> + 'repo_metadata', 'repo_settings', 'include_arches',
>>> + 'include_profiles', 'caches',
>>>   'repoman_incrementals', 'env', 'have', 'dev_keywords'
>>>   ],
>>>   'func_kwargs': {
>>> diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py
>>> index d980f4eca..0b1d74483 100644
>>> --- a/repoman/lib/repoman/modules/scan/depend/profile.py
>>> +++ b/repoman/lib/repoman/modules/scan/depend/profile.py
>>> @@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase):
>>>   @param options: cli options
>>>   @param repo_settings: repository settings instance
>>>   @param include_arches: set
>>> + @param include_profiles: set
>>>   @param caches: dictionary of our caches
>>>   @param repoman_incrementals: tuple
>>>   @param env: the environment
>>> @@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase):
>>>   self.options = kwargs.get('options')
>>>   self.repo_settings = kwargs.get('repo_settings')
>>>   self.include_arches = kwargs.get('include_arches')
>>> + self.include_profiles = kwargs.get('include_profiles')
>>>   self.caches = kwargs.get('caches')
>>>   self.repoman_incrementals = kwargs.get('repoman_incrementals')
>>>   self.env = kwargs.get('env')
>>> @@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase):
>>>   if arch not in self.include_arches:
>>>   continue
>>>  
>>> - relevant_profiles.extend(
>>> - (keyword, groups, prof) for prof in self.profiles[arch])
>>> + for prof in self.profiles[arch]:
>>> + if self.include_profiles is not None:
>>> + if prof not in self.include_profiles:
>>> + continue
>>> + relevant_profiles.append((keyword, groups, prof))
>>>  
>>>   relevant_profiles.sort(key=sort_key)
>>>  
>>> diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py
>>> index 1b3242a51..06234b0ad 100644
>>> --- a/repoman/lib/repoman/scanner.py
>>> +++ b/repoman/lib/repoman/scanner.py
>>> @@ -164,6 +164,10 @@ class Scanner(object):
>>>   if self.options.include_arches:
>>>   self.include_arches = set()
>>>   self.include_arches.update(*[x.split() for x in self.options.include_arches])
>>> + self.include_profiles = None
>>> + if self.options.include_profiles:
>>> + self.include_profiles = set()
>>> + self.include_profiles.update(*[x.split() for x in self.options.include_profiles])
>>>  
>>>   # Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and
>>>   # running `svn status` in every package dir will be too expensive.
>>> @@ -190,6 +194,7 @@ class Scanner(object):
>>>   "repo_metadata": self.repo_metadata,
>>>   "profiles": profiles,
>>>   "include_arches": self.include_arches,
>>> + "include_profiles": self.include_profiles,
>>>   "caches": self.caches,
>>>   "repoman_incrementals": self.repoman_incrementals,
>>>   "env": self.env,
>>> diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1
>>> index 766146f57..f33fb6098 100644
>>> --- a/repoman/man/repoman.1
>>> +++ b/repoman/man/repoman.1
>>> @@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode)
>>>  A space separated list of arches used to filter the selection of
>>>  profiles for dependency checks.
>>>  .TP
>>> +.BR "\-\-include\-profiles " PROFILES
>>> +A space separated list of profiles used to
>>> +define the selection of profiles for dependency checks.
>>> +.TP
>>>  \fB\-d\fR, \fB\-\-include\-dev\fR
>>>  Include dev profiles in dependency checks.
>>>  .TP
>>>  
>>
>> Looks good. Please merge.
>> --
>> Thanks,
>> Zac
>>
>
> Thank you!
>
> Does not look like I have permissions to do it:
Ok, merged with prof.sub_path fix:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=e9bb1e9681685f4e4d7174f51751356fd3f67d0c

--
Thanks,
Zac


signature.asc (1000 bytes) Download Attachment